117 	struct virtio_blk *vblk = req->q->queuedata;  in virtblk_scsi_request_done()  local
120 	sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);  in virtblk_scsi_request_done()
121 	sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);  in virtblk_scsi_request_done()
122 	sreq->result = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);  in virtblk_scsi_request_done()
129 	struct virtio_blk *vblk = disk->private_data;  in virtblk_ioctl()  local
134 	if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))  in virtblk_ioctl()
191 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
198 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
201 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
213 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
214 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
220 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
230 	BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);  in virtio_queue_rq()
252 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);  in virtio_queue_rq()
254 		0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));  in virtio_queue_rq()
255 	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));  in virtio_queue_rq()
262 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);  in virtio_queue_rq()
264 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);  in virtio_queue_rq()
267 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
269 		err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
271 		err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
273 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
275 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
283 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
285 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
288 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
296 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
297 	struct request_queue *q = vblk->disk->queue;  in virtblk_get_id()
309 	blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);  in virtblk_get_id()
319 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
322 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
323 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
325 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
327 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
377 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)  in virtblk_update_capacity()  argument
379 	struct virtio_device *vdev = vblk->vdev;  in virtblk_update_capacity()
380 	struct request_queue *q = vblk->disk->queue;  in virtblk_update_capacity()
404 		   vblk->disk->disk_name,  in virtblk_update_capacity()
411 	set_capacity(vblk->disk, capacity);  in virtblk_update_capacity()
416 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
420 	virtblk_update_capacity(vblk, true);  in virtblk_config_changed_work()
421 	revalidate_disk(vblk->disk);  in virtblk_config_changed_work()
422 	kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp);  in virtblk_config_changed_work()
427 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
429 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
432 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
440 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
449 	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);  in init_vq()
450 	if (!vblk->vqs)  in init_vq()
463 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
464 		names[i] = vblk->vqs[i].name;  in init_vq()
473 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
474 		vblk->vqs[i].vq = vqs[i];  in init_vq()
476 	vblk->num_vqs = num_vqs;  in init_vq()
483 		kfree(vblk->vqs);  in init_vq()
537 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
539 	blk_queue_write_cache(vblk->disk->queue, writeback, false);  in virtblk_update_cache_mode()
540 	revalidate_disk(vblk->disk);  in virtblk_update_cache_mode()
552 	struct virtio_blk *vblk = disk->private_data;  in virtblk_cache_type_store()  local
553 	struct virtio_device *vdev = vblk->vdev;  in virtblk_cache_type_store()
556 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in virtblk_cache_type_store()
571 	struct virtio_blk *vblk = disk->private_data;  in virtblk_cache_type_show()  local
572 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in virtblk_cache_type_show()
588 	struct virtio_blk *vblk = set->driver_data;  in virtblk_init_request()  local
594 	sg_init_table(vbr->sg, vblk->sg_elems);  in virtblk_init_request()
600 	struct virtio_blk *vblk = set->driver_data;  in virtblk_map_queues()  local
602 	return blk_mq_virtio_map_queues(set, vblk->vdev, 0);  in virtblk_map_queues()
629 	struct virtio_blk *vblk;  in virtblk_probe()  local
660 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
661 	if (!vblk) {  in virtblk_probe()
666 	vblk->vdev = vdev;  in virtblk_probe()
667 	vblk->sg_elems = sg_elems;  in virtblk_probe()
669 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
671 	err = init_vq(vblk);  in virtblk_probe()
676 	vblk->disk = alloc_disk(1 << PART_BITS);  in virtblk_probe()
677 	if (!vblk->disk) {  in virtblk_probe()
684 		virtblk_queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
690 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
691 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
692 	vblk->tag_set.queue_depth = virtblk_queue_depth;  in virtblk_probe()
693 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
694 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
695 	vblk->tag_set.cmd_size =  in virtblk_probe()
698 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
699 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
701 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
705 	q = blk_mq_init_queue(&vblk->tag_set);  in virtblk_probe()
710 	vblk->disk->queue = q;  in virtblk_probe()
712 	q->queuedata = vblk;  in virtblk_probe()
714 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
716 	vblk->disk->major = major;  in virtblk_probe()
717 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
718 	vblk->disk->private_data = vblk;  in virtblk_probe()
719 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
720 	vblk->disk->flags |= GENHD_FL_EXT_DEVT;  in virtblk_probe()
721 	vblk->index = index;  in virtblk_probe()
728 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
731 	blk_queue_max_segments(q, vblk->sg_elems-2);  in virtblk_probe()
780 	virtblk_update_capacity(vblk, false);  in virtblk_probe()
783 	device_add_disk(&vdev->dev, vblk->disk);  in virtblk_probe()
784 	err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial);  in virtblk_probe()
789 		err = device_create_file(disk_to_dev(vblk->disk),  in virtblk_probe()
792 		err = device_create_file(disk_to_dev(vblk->disk),  in virtblk_probe()
799 	del_gendisk(vblk->disk);  in virtblk_probe()
800 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_probe()
802 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
804 	put_disk(vblk->disk);  in virtblk_probe()
808 	kfree(vblk);  in virtblk_probe()
817 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
818 	int index = vblk->index;  in virtblk_remove()
822 	flush_work(&vblk->config_work);  in virtblk_remove()
824 	del_gendisk(vblk->disk);  in virtblk_remove()
825 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_remove()
827 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
832 	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);  in virtblk_remove()
833 	put_disk(vblk->disk);  in virtblk_remove()
835 	kfree(vblk->vqs);  in virtblk_remove()
836 	kfree(vblk);  in virtblk_remove()
846 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
852 	flush_work(&vblk->config_work);  in virtblk_freeze()
854 	blk_mq_quiesce_queue(vblk->disk->queue);  in virtblk_freeze()
862 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
871 	blk_mq_unquiesce_queue(vblk->disk->queue);  in virtblk_restore()