52 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,  in nvme_set_disk_name()  argument
56 		sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
57 	} else if (ns->head->disk) {  in nvme_set_disk_name()
59 				ctrl->instance, ns->head->instance);  in nvme_set_disk_name()
63 				ns->head->instance);  in nvme_set_disk_name()
69 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
73 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
74 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
75 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
91 		nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
92 		if (ns->ctrl->ana_log_buf) {  in nvme_failover_req()
93 			set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
94 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
102 		nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
109 		nvme_reset_ctrl(ns->ctrl);  in nvme_failover_req()
113 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
118 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
121 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_kick_requeue_lists()
122 		if (ns->head->disk)  in nvme_kick_requeue_lists()
123 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
137 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
139 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
147 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
158 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
162 	list_for_each_entry(ns, &ctrl->namespaces, list)  in nvme_mpath_clear_ctrl_paths()
163 		if (nvme_mpath_clear_current_path(ns))  in nvme_mpath_clear_ctrl_paths()
164 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
169 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
171 	return ns->ctrl->state != NVME_CTRL_LIVE ||  in nvme_path_is_disabled()
172 		test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
173 		test_bit(NVME_NS_REMOVING, &ns->flags);  in nvme_path_is_disabled()
179 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
181 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
182 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
186 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
190 		switch (ns->ana_state) {  in __nvme_find_path()
194 				found = ns;  in __nvme_find_path()
200 				fallback = ns;  in __nvme_find_path()
216 		struct nvme_ns *ns)  in nvme_next_ns()  argument
218 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
220 	if (ns)  in nvme_next_ns()
221 		return ns;  in nvme_next_ns()
228 	struct nvme_ns *ns, *found, *fallback = NULL;  in nvme_round_robin_path()  local
236 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
237 	     ns != old;  in nvme_round_robin_path()
238 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
239 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
242 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
243 			found = ns;  in nvme_round_robin_path()
246 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
247 			fallback = ns;  in nvme_round_robin_path()
258 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
260 	return ns->ctrl->state == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
261 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
267 	struct nvme_ns *ns;  in nvme_find_path()  local
269 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
270 	if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns)  in nvme_find_path()
271 		ns = nvme_round_robin_path(head, node, ns);  in nvme_find_path()
272 	if (unlikely(!ns || !nvme_path_is_optimized(ns)))  in nvme_find_path()
273 		ns = __nvme_find_path(head, node);  in nvme_find_path()
274 	return ns;  in nvme_find_path()
279 	struct nvme_ns *ns;  in nvme_available_path()  local
281 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
282 		switch (ns->ctrl->state) {  in nvme_available_path()
300 	struct nvme_ns *ns;  in nvme_ns_head_make_request()  local
313 	ns = nvme_find_path(head);  in nvme_ns_head_make_request()
314 	if (likely(ns)) {  in nvme_ns_head_make_request()
315 		bio->bi_disk = ns->disk;  in nvme_ns_head_make_request()
318 				      disk_devt(ns->head->disk),  in nvme_ns_head_make_request()
411 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
413 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
415 	lockdep_assert_held(&ns->head->lock);  in nvme_mpath_set_live()
424 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
433 	synchronize_srcu(&ns->head->srcu);  in nvme_mpath_set_live()
434 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_set_live()
483 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
485 	mutex_lock(&ns->head->lock);  in nvme_update_ns_ana_state()
486 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
487 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
488 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
490 	if (nvme_state_is_live(ns->ana_state))  in nvme_update_ns_ana_state()
491 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
492 	mutex_unlock(&ns->head->lock);  in nvme_update_ns_ana_state()
500 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
513 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_update_ana_state()
516 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
518 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
519 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
636 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
638 	return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
645 	struct nvme_ns *ns = data;  in nvme_set_ns_ana_state()  local
647 	if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {  in nvme_set_ns_ana_state()
648 		nvme_update_ns_ana_state(desc, ns);  in nvme_set_ns_ana_state()
655 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)  in nvme_mpath_add_disk()  argument
657 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
658 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
659 		ns->ana_grpid = le32_to_cpu(id->anagrpid);  in nvme_mpath_add_disk()
660 		nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);  in nvme_mpath_add_disk()
661 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
663 		mutex_lock(&ns->head->lock);  in nvme_mpath_add_disk()
664 		ns->ana_state = NVME_ANA_OPTIMIZED;   in nvme_mpath_add_disk()
665 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
666 		mutex_unlock(&ns->head->lock);  in nvme_mpath_add_disk()