85 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
90 	nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
97 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
98 		set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
99 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
102 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
104 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
110 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
111 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
114 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
119 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
122 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_kick_requeue_lists()
123 		if (!ns->head->disk)  in nvme_kick_requeue_lists()
125 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
127 			disk_uevent(ns->head->disk, KOBJ_CHANGE);  in nvme_kick_requeue_lists()
141 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
143 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
151 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
162 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
165 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_mpath_clear_ctrl_paths()
166 		nvme_mpath_clear_current_path(ns);  in nvme_mpath_clear_ctrl_paths()
167 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
172 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)  in nvme_mpath_revalidate_paths()  argument
174 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()
180 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_mpath_revalidate_paths()
181 		if (capacity != get_capacity(ns->disk))  in nvme_mpath_revalidate_paths()
182 			clear_bit(NVME_NS_READY, &ns->flags);  in nvme_mpath_revalidate_paths()
191 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
198 	if (ns->ctrl->state != NVME_CTRL_LIVE &&  in nvme_path_is_disabled()
199 	    ns->ctrl->state != NVME_CTRL_DELETING)  in nvme_path_is_disabled()
201 	if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
202 	    !test_bit(NVME_NS_READY, &ns->flags))  in nvme_path_is_disabled()
210 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
212 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
213 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
217 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
221 		switch (ns->ana_state) {  in __nvme_find_path()
225 				found = ns;  in __nvme_find_path()
231 				fallback = ns;  in __nvme_find_path()
247 		struct nvme_ns *ns)  in nvme_next_ns()  argument
249 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
251 	if (ns)  in nvme_next_ns()
252 		return ns;  in nvme_next_ns()
259 	struct nvme_ns *ns, *found = NULL;  in nvme_round_robin_path()  local
267 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
268 	     ns && ns != old;  in nvme_round_robin_path()
269 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
270 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
273 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
274 			found = ns;  in nvme_round_robin_path()
277 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
278 			found = ns;  in nvme_round_robin_path()
299 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
301 	return ns->ctrl->state == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
302 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
308 	struct nvme_ns *ns;  in nvme_find_path()  local
310 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
311 	if (unlikely(!ns))  in nvme_find_path()
315 		return nvme_round_robin_path(head, node, ns);  in nvme_find_path()
316 	if (unlikely(!nvme_path_is_optimized(ns)))  in nvme_find_path()
318 	return ns;  in nvme_find_path()
323 	struct nvme_ns *ns;  in nvme_available_path()  local
325 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
326 		if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))  in nvme_available_path()
328 		switch (ns->ctrl->state) {  in nvme_available_path()
345 	struct nvme_ns *ns;  in nvme_ns_head_submit_bio()  local
356 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
357 	if (likely(ns)) {  in nvme_ns_head_submit_bio()
358 		bio_set_dev(bio, ns->disk->part0);  in nvme_ns_head_submit_bio()
360 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
395 	struct nvme_ns *ns;  in nvme_ns_head_report_zones()  local
399 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
400 	if (ns)  in nvme_ns_head_report_zones()
401 		ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);  in nvme_ns_head_report_zones()
531 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
533 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
548 			clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags);  in nvme_mpath_set_live()
555 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
619 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
621 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
622 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
623 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
633 	if (nvme_state_is_live(ns->ana_state) &&  in nvme_update_ns_ana_state()
634 	    ns->ctrl->state == NVME_CTRL_LIVE)  in nvme_update_ns_ana_state()
635 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
643 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
656 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_update_ana_state()
660 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
662 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
663 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
666 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
792 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
794 	return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
810 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)  in nvme_mpath_add_disk()  argument
812 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
818 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
819 		ns->ana_grpid = le32_to_cpu(anagrpid);  in nvme_mpath_add_disk()
820 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
821 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
824 			nvme_update_ns_ana_state(&desc, ns);  in nvme_mpath_add_disk()
827 			set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_mpath_add_disk()
828 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_mpath_add_disk()
831 		ns->ana_state = NVME_ANA_OPTIMIZED;  in nvme_mpath_add_disk()
832 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
835 	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
837 				   ns->head->disk->queue);  in nvme_mpath_add_disk()
839 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
840 		ns->head->disk->nr_zones = ns->disk->nr_zones;  in nvme_mpath_add_disk()