57 	if (!ns->head->disk) {  in nvme_mpath_set_disk_name()
59 			ns->head->instance);  in nvme_mpath_set_disk_name()
63 		ns->ctrl->instance, ns->head->instance);  in nvme_mpath_set_disk_name()
87 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
89 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
90 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
91 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
94 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
103 		if (ns->head->disk)  in nvme_kick_requeue_lists()
104 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
120 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()  local
124 	if (!head)  in nvme_mpath_clear_current_path()
128 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
129 			rcu_assign_pointer(head->current_path[node], NULL);  in nvme_mpath_clear_current_path()
145 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
152 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()  local
153 	sector_t capacity = get_capacity(head->disk);  in nvme_mpath_revalidate_paths()
156 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_mpath_revalidate_paths()
162 		rcu_assign_pointer(head->current_path[node], NULL);  in nvme_mpath_revalidate_paths()
181 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)  in __nvme_find_path()  argument
186 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
190 		if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)  in __nvme_find_path()
216 		rcu_assign_pointer(head->current_path[node], found);  in __nvme_find_path()
220 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,  in nvme_next_ns()  argument
223 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
227 	return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);  in nvme_next_ns()
230 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,  in nvme_round_robin_path()  argument
235 	if (list_is_singular(&head->list)) {  in nvme_round_robin_path()
241 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
243 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
269 	rcu_assign_pointer(head->current_path[node], found);  in nvme_round_robin_path()
279 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)  in nvme_find_path()  argument
284 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
286 		return __nvme_find_path(head, node);  in nvme_find_path()
288 	if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)  in nvme_find_path()
289 		return nvme_round_robin_path(head, node, ns);  in nvme_find_path()
291 		return __nvme_find_path(head, node);  in nvme_find_path()
295 static bool nvme_available_path(struct nvme_ns_head *head)  in nvme_available_path()  argument
299 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
317 	struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;  in nvme_ns_head_submit_bio()  local
318 	struct device *dev = disk_to_dev(head->disk);  in nvme_ns_head_submit_bio()
330 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_submit_bio()
331 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
335 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
338 	} else if (nvme_available_path(head)) {  in nvme_ns_head_submit_bio()
341 		spin_lock_irq(&head->requeue_lock);  in nvme_ns_head_submit_bio()
342 		bio_list_add(&head->requeue_list, bio);  in nvme_ns_head_submit_bio()
343 		spin_unlock_irq(&head->requeue_lock);  in nvme_ns_head_submit_bio()
351 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_submit_bio()
371 	struct nvme_ns_head *head = disk->private_data;  in nvme_ns_head_report_zones()  local
375 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_report_zones()
376 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
379 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_report_zones()
423 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)  in nvme_add_ns_head_cdev()  argument
427 	head->cdev_device.parent = &head->subsys->dev;  in nvme_add_ns_head_cdev()
428 	ret = dev_set_name(&head->cdev_device, "ng%dn%d",  in nvme_add_ns_head_cdev()
429 			   head->subsys->instance, head->instance);  in nvme_add_ns_head_cdev()
432 	ret = nvme_cdev_add(&head->cdev, &head->cdev_device,  in nvme_add_ns_head_cdev()
439 	struct nvme_ns_head *head =  in nvme_requeue_work()  local
443 	spin_lock_irq(&head->requeue_lock);  in nvme_requeue_work()
444 	next = bio_list_get(&head->requeue_list);  in nvme_requeue_work()
445 	spin_unlock_irq(&head->requeue_lock);  in nvme_requeue_work()
455 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)  in nvme_mpath_alloc_disk()  argument
459 	mutex_init(&head->lock);  in nvme_mpath_alloc_disk()
460 	bio_list_init(&head->requeue_list);  in nvme_mpath_alloc_disk()
461 	spin_lock_init(&head->requeue_lock);  in nvme_mpath_alloc_disk()
462 	INIT_WORK(&head->requeue_work, nvme_requeue_work);  in nvme_mpath_alloc_disk()
472 	head->disk = blk_alloc_disk(ctrl->numa_node);  in nvme_mpath_alloc_disk()
473 	if (!head->disk)  in nvme_mpath_alloc_disk()
475 	head->disk->fops = &nvme_ns_head_ops;  in nvme_mpath_alloc_disk()
476 	head->disk->private_data = head;  in nvme_mpath_alloc_disk()
477 	sprintf(head->disk->disk_name, "nvme%dn%d",  in nvme_mpath_alloc_disk()
478 			ctrl->subsys->instance, head->instance);  in nvme_mpath_alloc_disk()
480 	blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue);  in nvme_mpath_alloc_disk()
481 	blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue);  in nvme_mpath_alloc_disk()
484 	blk_queue_logical_block_size(head->disk->queue, 512);  in nvme_mpath_alloc_disk()
485 	blk_set_stacking_limits(&head->disk->queue->limits);  in nvme_mpath_alloc_disk()
490 	blk_queue_write_cache(head->disk->queue, vwc, vwc);  in nvme_mpath_alloc_disk()
496 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()  local
498 	if (!head->disk)  in nvme_mpath_set_live()
501 	if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {  in nvme_mpath_set_live()
502 		device_add_disk(&head->subsys->dev, head->disk,  in nvme_mpath_set_live()
504 		nvme_add_ns_head_cdev(head);  in nvme_mpath_set_live()
507 	mutex_lock(&head->lock);  in nvme_mpath_set_live()
511 		srcu_idx = srcu_read_lock(&head->srcu);  in nvme_mpath_set_live()
513 			__nvme_find_path(head, node);  in nvme_mpath_set_live()
514 		srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_mpath_set_live()
516 	mutex_unlock(&head->lock);  in nvme_mpath_set_live()
518 	synchronize_srcu(&head->srcu);  in nvme_mpath_set_live()
519 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_set_live()
604 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
606 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
610 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
772 	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
774 				   ns->head->disk->queue);  in nvme_mpath_add_disk()
776 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
777 		ns->head->disk->queue->nr_zones = ns->queue->nr_zones;  in nvme_mpath_add_disk()
781 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)  in nvme_mpath_shutdown_disk()  argument
783 	if (!head->disk)  in nvme_mpath_shutdown_disk()
785 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_shutdown_disk()
786 	if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {  in nvme_mpath_shutdown_disk()
787 		nvme_cdev_del(&head->cdev, &head->cdev_device);  in nvme_mpath_shutdown_disk()
788 		del_gendisk(head->disk);  in nvme_mpath_shutdown_disk()
792 void nvme_mpath_remove_disk(struct nvme_ns_head *head)  in nvme_mpath_remove_disk()  argument
794 	if (!head->disk)  in nvme_mpath_remove_disk()
796 	blk_set_queue_dying(head->disk->queue);  in nvme_mpath_remove_disk()
798 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_remove_disk()
799 	flush_work(&head->requeue_work);  in nvme_mpath_remove_disk()
800 	blk_cleanup_disk(head->disk);  in nvme_mpath_remove_disk()