53 bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, int *flags)  in nvme_mpath_set_disk_name()  argument
57 	if (!ns->head->disk) {  in nvme_mpath_set_disk_name()
58 		sprintf(disk_name, "nvme%dn%d", ns->ctrl->subsys->instance,  in nvme_mpath_set_disk_name()
59 			ns->head->instance);  in nvme_mpath_set_disk_name()
62 	sprintf(disk_name, "nvme%dc%dn%d", ns->ctrl->subsys->instance,  in nvme_mpath_set_disk_name()
63 		ns->ctrl->instance, ns->head->instance);  in nvme_mpath_set_disk_name()
70 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
75 	nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
82 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
83 		set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
84 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
87 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
89 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
90 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
91 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
94 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
99 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
102 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_kick_requeue_lists()
103 		if (ns->head->disk)  in nvme_kick_requeue_lists()
104 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
118 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
120 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
128 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
139 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
143 	list_for_each_entry(ns, &ctrl->namespaces, list)  in nvme_mpath_clear_ctrl_paths()
144 		if (nvme_mpath_clear_current_path(ns))  in nvme_mpath_clear_ctrl_paths()
145 			kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
150 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)  in nvme_mpath_revalidate_paths()  argument
152 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()
156 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_mpath_revalidate_paths()
157 		if (capacity != get_capacity(ns->disk))  in nvme_mpath_revalidate_paths()
158 			clear_bit(NVME_NS_READY, &ns->flags);  in nvme_mpath_revalidate_paths()
165 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
172 	if (ns->ctrl->state != NVME_CTRL_LIVE &&  in nvme_path_is_disabled()
173 	    ns->ctrl->state != NVME_CTRL_DELETING)  in nvme_path_is_disabled()
175 	if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
176 	    !test_bit(NVME_NS_READY, &ns->flags))  in nvme_path_is_disabled()
184 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
186 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
187 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
191 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
195 		switch (ns->ana_state) {  in __nvme_find_path()
199 				found = ns;  in __nvme_find_path()
205 				fallback = ns;  in __nvme_find_path()
221 		struct nvme_ns *ns)  in nvme_next_ns()  argument
223 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
225 	if (ns)  in nvme_next_ns()
226 		return ns;  in nvme_next_ns()
233 	struct nvme_ns *ns, *found = NULL;  in nvme_round_robin_path()  local
241 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
242 	     ns && ns != old;  in nvme_round_robin_path()
243 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
244 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
247 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
248 			found = ns;  in nvme_round_robin_path()
251 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
252 			found = ns;  in nvme_round_robin_path()
273 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
275 	return ns->ctrl->state == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
276 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
282 	struct nvme_ns *ns;  in nvme_find_path()  local
284 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
285 	if (unlikely(!ns))  in nvme_find_path()
289 		return nvme_round_robin_path(head, node, ns);  in nvme_find_path()
290 	if (unlikely(!nvme_path_is_optimized(ns)))  in nvme_find_path()
292 	return ns;  in nvme_find_path()
297 	struct nvme_ns *ns;  in nvme_available_path()  local
299 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
300 		if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))  in nvme_available_path()
302 		switch (ns->ctrl->state) {  in nvme_available_path()
319 	struct nvme_ns *ns;  in nvme_ns_head_submit_bio()  local
331 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
332 	if (likely(ns)) {  in nvme_ns_head_submit_bio()
333 		bio_set_dev(bio, ns->disk->part0);  in nvme_ns_head_submit_bio()
335 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
372 	struct nvme_ns *ns;  in nvme_ns_head_report_zones()  local
376 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
377 	if (ns)  in nvme_ns_head_report_zones()
378 		ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);  in nvme_ns_head_report_zones()
494 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
496 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
508 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
572 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
574 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
575 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
576 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
578 	if (nvme_state_is_live(ns->ana_state))  in nvme_update_ns_ana_state()
579 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
587 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
600 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_update_ana_state()
604 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
606 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
607 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
610 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
729 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
731 	return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
747 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)  in nvme_mpath_add_disk()  argument
749 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
755 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
756 		ns->ana_grpid = le32_to_cpu(id->anagrpid);  in nvme_mpath_add_disk()
757 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
758 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
761 			nvme_update_ns_ana_state(&desc, ns);  in nvme_mpath_add_disk()
764 			set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_mpath_add_disk()
765 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_mpath_add_disk()
768 		ns->ana_state = NVME_ANA_OPTIMIZED;  in nvme_mpath_add_disk()
769 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
772 	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
774 				   ns->head->disk->queue);  in nvme_mpath_add_disk()
776 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
777 		ns->head->disk->queue->nr_zones = ns->queue->nr_zones;  in nvme_mpath_add_disk()