53 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,  in nvme_set_disk_name()  argument
57 		sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
58 	} else if (ns->head->disk) {  in nvme_set_disk_name()
60 				ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
64 				ns->head->instance);  in nvme_set_disk_name()
70 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
74 	nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
81 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
82 		set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
83 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
86 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
87 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
88 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
91 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
96 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
99 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_kick_requeue_lists()
100 		if (ns->head->disk)  in nvme_kick_requeue_lists()
101 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
115 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
117 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
125 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
136 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
140 	list_for_each_entry(ns, &ctrl->namespaces, list)  in nvme_mpath_clear_ctrl_paths()
141 		if (nvme_mpath_clear_current_path(ns))  in nvme_mpath_clear_ctrl_paths()
142 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
147 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
154 	if (ns->ctrl->state != NVME_CTRL_LIVE &&  in nvme_path_is_disabled()
155 	    ns->ctrl->state != NVME_CTRL_DELETING)  in nvme_path_is_disabled()
157 	if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
158 	    test_bit(NVME_NS_REMOVING, &ns->flags))  in nvme_path_is_disabled()
166 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
168 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
169 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
173 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
177 		switch (ns->ana_state) {  in __nvme_find_path()
181 				found = ns;  in __nvme_find_path()
187 				fallback = ns;  in __nvme_find_path()
203 		struct nvme_ns *ns)  in nvme_next_ns()  argument
205 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
207 	if (ns)  in nvme_next_ns()
208 		return ns;  in nvme_next_ns()
215 	struct nvme_ns *ns, *found = NULL;  in nvme_round_robin_path()  local
223 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
224 	     ns != old;  in nvme_round_robin_path()
225 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
226 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
229 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
230 			found = ns;  in nvme_round_robin_path()
233 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
234 			found = ns;  in nvme_round_robin_path()
255 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
257 	return ns->ctrl->state == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
258 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
264 	struct nvme_ns *ns;  in nvme_find_path()  local
266 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
267 	if (unlikely(!ns))  in nvme_find_path()
271 		return nvme_round_robin_path(head, node, ns);  in nvme_find_path()
272 	if (unlikely(!nvme_path_is_optimized(ns)))  in nvme_find_path()
274 	return ns;  in nvme_find_path()
279 	struct nvme_ns *ns;  in nvme_available_path()  local
281 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
282 		switch (ns->ctrl->state) {  in nvme_available_path()
299 	struct nvme_ns *ns;  in nvme_ns_head_submit_bio()  local
311 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
312 	if (likely(ns)) {  in nvme_ns_head_submit_bio()
313 		bio->bi_disk = ns->disk;  in nvme_ns_head_submit_bio()
316 				      disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
407 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
409 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
419 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
483 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
485 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
486 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
487 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
489 	if (nvme_state_is_live(ns->ana_state))  in nvme_update_ns_ana_state()
490 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
498 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
511 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_update_ana_state()
514 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
516 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
517 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
637 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
639 	return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
655 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)  in nvme_mpath_add_disk()  argument
657 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
663 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
664 		ns->ana_grpid = le32_to_cpu(id->anagrpid);  in nvme_mpath_add_disk()
665 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
666 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
669 			nvme_update_ns_ana_state(&desc, ns);  in nvme_mpath_add_disk()
672 		ns->ana_state = NVME_ANA_OPTIMIZED;   in nvme_mpath_add_disk()
673 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
676 	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
678 				   ns->head->disk->queue);  in nvme_mpath_add_disk()