39 		sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
40 	} else if (ns->head->disk) {  in nvme_set_disk_name()
42 				ctrl->cntlid, ns->head->instance);  in nvme_set_disk_name()
46 				ns->head->instance);  in nvme_set_disk_name()
56 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
57 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
58 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
89 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
98 		if (ns->head->disk)  in nvme_kick_requeue_lists()
99 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
113 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head)  in __nvme_find_path()  argument
117 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
123 			rcu_assign_pointer(head->current_path, ns);  in __nvme_find_path()
134 		rcu_assign_pointer(head->current_path, fallback);  in __nvme_find_path()
144 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)  in nvme_find_path()  argument
146 	struct nvme_ns *ns = srcu_dereference(head->current_path, &head->srcu);  in nvme_find_path()
149 		ns = __nvme_find_path(head);  in nvme_find_path()
156 	struct nvme_ns_head *head = q->queuedata;  in nvme_ns_head_make_request()  local
157 	struct device *dev = disk_to_dev(head->disk);  in nvme_ns_head_make_request()
162 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_make_request()
163 	ns = nvme_find_path(head);  in nvme_ns_head_make_request()
168 				      disk_devt(ns->head->disk),  in nvme_ns_head_make_request()
171 	} else if (!list_empty_careful(&head->list)) {  in nvme_ns_head_make_request()
174 		spin_lock_irq(&head->requeue_lock);  in nvme_ns_head_make_request()
175 		bio_list_add(&head->requeue_list, bio);  in nvme_ns_head_make_request()
176 		spin_unlock_irq(&head->requeue_lock);  in nvme_ns_head_make_request()
184 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_make_request()
190 	struct nvme_ns_head *head = q->queuedata;  in nvme_ns_head_poll()  local
195 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_poll()
196 	ns = srcu_dereference(head->current_path, &head->srcu);  in nvme_ns_head_poll()
199 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_poll()
205 	struct nvme_ns_head *head =  in nvme_requeue_work()  local
209 	spin_lock_irq(&head->requeue_lock);  in nvme_requeue_work()
210 	next = bio_list_get(&head->requeue_list);  in nvme_requeue_work()
211 	spin_unlock_irq(&head->requeue_lock);  in nvme_requeue_work()
221 		bio->bi_disk = head->disk;  in nvme_requeue_work()
226 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)  in nvme_mpath_alloc_disk()  argument
231 	mutex_init(&head->lock);  in nvme_mpath_alloc_disk()
232 	bio_list_init(&head->requeue_list);  in nvme_mpath_alloc_disk()
233 	spin_lock_init(&head->requeue_lock);  in nvme_mpath_alloc_disk()
234 	INIT_WORK(&head->requeue_work, nvme_requeue_work);  in nvme_mpath_alloc_disk()
247 	q->queuedata = head;  in nvme_mpath_alloc_disk()
259 	head->disk = alloc_disk(0);  in nvme_mpath_alloc_disk()
260 	if (!head->disk)  in nvme_mpath_alloc_disk()
262 	head->disk->fops = &nvme_ns_head_ops;  in nvme_mpath_alloc_disk()
263 	head->disk->private_data = head;  in nvme_mpath_alloc_disk()
264 	head->disk->queue = q;  in nvme_mpath_alloc_disk()
265 	head->disk->flags = GENHD_FL_EXT_DEVT;  in nvme_mpath_alloc_disk()
266 	sprintf(head->disk->disk_name, "nvme%dn%d",  in nvme_mpath_alloc_disk()
267 			ctrl->subsys->instance, head->instance);  in nvme_mpath_alloc_disk()
278 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()  local
280 	lockdep_assert_held(&ns->head->lock);  in nvme_mpath_set_live()
282 	if (!head->disk)  in nvme_mpath_set_live()
285 	if (!(head->disk->flags & GENHD_FL_UP)) {  in nvme_mpath_set_live()
286 		device_add_disk(&head->subsys->dev, head->disk);  in nvme_mpath_set_live()
287 		if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,  in nvme_mpath_set_live()
289 			dev_warn(&head->subsys->dev,  in nvme_mpath_set_live()
293 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_set_live()
346 	mutex_lock(&ns->head->lock);  in nvme_update_ns_ana_state()
354 	mutex_unlock(&ns->head->lock);  in nvme_update_ns_ana_state()
376 		if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))  in nvme_update_ana_state()
486 		mutex_lock(&ns->head->lock);  in nvme_mpath_add_disk()
489 		mutex_unlock(&ns->head->lock);  in nvme_mpath_add_disk()
493 void nvme_mpath_remove_disk(struct nvme_ns_head *head)  in nvme_mpath_remove_disk()  argument
495 	if (!head->disk)  in nvme_mpath_remove_disk()
497 	if (head->disk->flags & GENHD_FL_UP) {  in nvme_mpath_remove_disk()
498 		sysfs_remove_group(&disk_to_dev(head->disk)->kobj,  in nvme_mpath_remove_disk()
500 		del_gendisk(head->disk);  in nvme_mpath_remove_disk()
502 	blk_set_queue_dying(head->disk->queue);  in nvme_mpath_remove_disk()
504 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_remove_disk()
505 	flush_work(&head->requeue_work);  in nvme_mpath_remove_disk()
506 	blk_cleanup_queue(head->disk->queue);  in nvme_mpath_remove_disk()
507 	put_disk(head->disk);  in nvme_mpath_remove_disk()