56 		sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
57 	} else if (ns->head->disk) {  in nvme_set_disk_name()
59 				ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
63 				ns->head->instance);  in nvme_set_disk_name()
73 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
74 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
75 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
113 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
122 		if (ns->head->disk)  in nvme_kick_requeue_lists()
123 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
139 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()  local
143 	if (!head)  in nvme_mpath_clear_current_path()
147 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
148 			rcu_assign_pointer(head->current_path[node], NULL);  in nvme_mpath_clear_current_path()
164 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
176 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)  in __nvme_find_path()  argument
181 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
185 		if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)  in __nvme_find_path()
211 		rcu_assign_pointer(head->current_path[node], found);  in __nvme_find_path()
215 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,  in nvme_next_ns()  argument
218 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
222 	return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);  in nvme_next_ns()
225 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,  in nvme_round_robin_path()  argument
230 	if (list_is_singular(&head->list)) {  in nvme_round_robin_path()
236 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
238 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
254 	rcu_assign_pointer(head->current_path[node], found);  in nvme_round_robin_path()
264 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)  in nvme_find_path()  argument
269 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
270 	if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns)  in nvme_find_path()
271 		ns = nvme_round_robin_path(head, node, ns);  in nvme_find_path()
273 		ns = __nvme_find_path(head, node);  in nvme_find_path()
277 static bool nvme_available_path(struct nvme_ns_head *head)  in nvme_available_path()  argument
281 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
298 	struct nvme_ns_head *head = q->queuedata;  in nvme_ns_head_make_request()  local
299 	struct device *dev = disk_to_dev(head->disk);  in nvme_ns_head_make_request()
312 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_make_request()
313 	ns = nvme_find_path(head);  in nvme_ns_head_make_request()
318 				      disk_devt(ns->head->disk),  in nvme_ns_head_make_request()
321 	} else if (nvme_available_path(head)) {  in nvme_ns_head_make_request()
324 		spin_lock_irq(&head->requeue_lock);  in nvme_ns_head_make_request()
325 		bio_list_add(&head->requeue_list, bio);  in nvme_ns_head_make_request()
326 		spin_unlock_irq(&head->requeue_lock);  in nvme_ns_head_make_request()
334 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_make_request()
340 	struct nvme_ns_head *head =  in nvme_requeue_work()  local
344 	spin_lock_irq(&head->requeue_lock);  in nvme_requeue_work()
345 	next = bio_list_get(&head->requeue_list);  in nvme_requeue_work()
346 	spin_unlock_irq(&head->requeue_lock);  in nvme_requeue_work()
356 		bio->bi_disk = head->disk;  in nvme_requeue_work()
361 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)  in nvme_mpath_alloc_disk()  argument
366 	mutex_init(&head->lock);  in nvme_mpath_alloc_disk()
367 	bio_list_init(&head->requeue_list);  in nvme_mpath_alloc_disk()
368 	spin_lock_init(&head->requeue_lock);  in nvme_mpath_alloc_disk()
369 	INIT_WORK(&head->requeue_work, nvme_requeue_work);  in nvme_mpath_alloc_disk()
382 	q->queuedata = head;  in nvme_mpath_alloc_disk()
394 	head->disk = alloc_disk(0);  in nvme_mpath_alloc_disk()
395 	if (!head->disk)  in nvme_mpath_alloc_disk()
397 	head->disk->fops = &nvme_ns_head_ops;  in nvme_mpath_alloc_disk()
398 	head->disk->private_data = head;  in nvme_mpath_alloc_disk()
399 	head->disk->queue = q;  in nvme_mpath_alloc_disk()
400 	head->disk->flags = GENHD_FL_EXT_DEVT;  in nvme_mpath_alloc_disk()
401 	sprintf(head->disk->disk_name, "nvme%dn%d",  in nvme_mpath_alloc_disk()
402 			ctrl->subsys->instance, head->instance);  in nvme_mpath_alloc_disk()
413 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()  local
415 	lockdep_assert_held(&ns->head->lock);  in nvme_mpath_set_live()
417 	if (!head->disk)  in nvme_mpath_set_live()
420 	if (!(head->disk->flags & GENHD_FL_UP))  in nvme_mpath_set_live()
421 		device_add_disk(&head->subsys->dev, head->disk,  in nvme_mpath_set_live()
427 		srcu_idx = srcu_read_lock(&head->srcu);  in nvme_mpath_set_live()
429 			__nvme_find_path(head, node);  in nvme_mpath_set_live()
430 		srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_mpath_set_live()
433 	synchronize_srcu(&ns->head->srcu);  in nvme_mpath_set_live()
434 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_set_live()
485 	mutex_lock(&ns->head->lock);  in nvme_update_ns_ana_state()
492 	mutex_unlock(&ns->head->lock);  in nvme_update_ns_ana_state()
516 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
518 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
663 		mutex_lock(&ns->head->lock);  in nvme_mpath_add_disk()
666 		mutex_unlock(&ns->head->lock);  in nvme_mpath_add_disk()
670 void nvme_mpath_remove_disk(struct nvme_ns_head *head)  in nvme_mpath_remove_disk()  argument
672 	if (!head->disk)  in nvme_mpath_remove_disk()
674 	if (head->disk->flags & GENHD_FL_UP)  in nvme_mpath_remove_disk()
675 		del_gendisk(head->disk);  in nvme_mpath_remove_disk()
676 	blk_set_queue_dying(head->disk->queue);  in nvme_mpath_remove_disk()
678 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_remove_disk()
679 	flush_work(&head->requeue_work);  in nvme_mpath_remove_disk()
680 	blk_cleanup_queue(head->disk->queue);  in nvme_mpath_remove_disk()
681 	put_disk(head->disk);  in nvme_mpath_remove_disk()