123 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
125 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
128 void nvme_queue_scan(struct nvme_ctrl *ctrl)  in nvme_queue_scan()
143 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)  in nvme_try_sched_reset()
155 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),  in nvme_failfast_work()
156 			struct nvme_ctrl, failfast_work);  in nvme_failfast_work()
166 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)  in nvme_start_failfast_work()
175 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)  in nvme_stop_failfast_work()
185 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)  in nvme_reset_ctrl()
195 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)  in nvme_reset_ctrl_sync()
209 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)  in nvme_do_delete_ctrl()
223 	struct nvme_ctrl *ctrl =  in nvme_delete_ctrl_work()
224 		container_of(work, struct nvme_ctrl, delete_work);  in nvme_delete_ctrl_work()
229 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)  in nvme_delete_ctrl()
239 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)  in nvme_delete_ctrl_sync()
392 	struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;  in nvme_complete_rq()
447 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,  in nvme_cancel_request()
461 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)  in nvme_cancel_tagset()
471 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)  in nvme_cancel_admin_tagset()
481 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,  in nvme_change_ctrl_state()
581 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)  in nvme_state_terminal()
603 bool nvme_wait_reset(struct nvme_ctrl *ctrl)  in nvme_wait_reset()
692 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,  in nvme_fail_nonready_command()
705 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,  in __nvme_check_ready()
931 		struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;  in nvme_cleanup_cmd()
1072 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)  in nvme_command_effects()
1094 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,  in nvme_passthru_start()
1114 void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,  in nvme_passthru_end()
1156 	struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;  in nvme_execute_passthru_rq()
1170 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)  in nvme_queue_keep_alive_work()
1178 	struct nvme_ctrl *ctrl = rq->end_io_data;  in nvme_keep_alive_end_io()
1204 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),  in nvme_keep_alive_work()
1205 			struct nvme_ctrl, ka_work);  in nvme_keep_alive_work()
1233 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)  in nvme_start_keep_alive()
1241 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)  in nvme_stop_keep_alive()
1250 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,  in nvme_update_keep_alive()
1271 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)  in nvme_ctrl_limited_cns()
1278 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)  in nvme_identify_ctrl()
1298 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,  in nvme_process_ns_desc()
1350 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,  in nvme_identify_ns_descs()
1404 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,  in nvme_identify_ns()
1435 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,  in nvme_ns_info_from_identify()
1464 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,  in nvme_ns_info_from_id_cs_indep()
1490 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,  in nvme_features()
1508 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,  in nvme_set_features()
1517 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,  in nvme_get_features()
1526 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)  in nvme_set_queue_count()
1558 static void nvme_enable_aen(struct nvme_ctrl *ctrl)  in nvme_enable_aen()
1680 	struct nvme_ctrl *ctrl = ns->ctrl;  in nvme_config_discard()
1720 	struct nvme_ctrl *ctrl = ns->ctrl;  in nvme_init_ms()
1778 	struct nvme_ctrl *ctrl = ns->ctrl;  in nvme_configure_metadata()
1823 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,  in nvme_set_queue_limits()
1924 	struct nvme_ctrl *ctrl = ns->ctrl;  in nvme_set_chunk_sectors()
2185 	struct nvme_ctrl *ctrl = data;  in nvme_sec_submit()
2224 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)  in nvme_wait_ready()
2256 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)  in nvme_disable_ctrl()
2274 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)  in nvme_enable_ctrl()
2339 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)  in nvme_shutdown_ctrl()
2370 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)  in nvme_configure_timestamp()
2387 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)  in nvme_configure_host_options()
2468 static int nvme_configure_apst(struct nvme_ctrl *ctrl)  in nvme_configure_apst()
2575 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_set_latency_tolerance()
2686 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,  in nvme_init_subnqn()
2844 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)  in nvme_discovery_ctrl()
2850 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)  in nvme_validate_cntlid()
2852 	struct nvme_ctrl *tmp;  in nvme_validate_cntlid()
2880 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)  in nvme_init_subsystem()
2967 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,  in nvme_get_log()
2986 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,  in nvme_get_effects_log()
3012 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)  in nvme_mps_to_sectors()
3021 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)  in nvme_init_non_mdts_limits()
3073 static int nvme_init_identify(struct nvme_ctrl *ctrl)  in nvme_init_identify()
3232 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)  in nvme_init_ctrl_finish()
3281 	struct nvme_ctrl *ctrl =  in nvme_dev_open()
3282 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);  in nvme_dev_open()
3303 	struct nvme_ctrl *ctrl =  in nvme_dev_release()
3304 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);  in nvme_dev_release()
3324 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_reset()
3338 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_rescan()
3481         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3495         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3510 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_delete()
3522 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_transport()
3532 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_state()
3556 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_subsysnqn()
3566 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_hostnqn()
3576 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_hostid()
3586 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_sysfs_show_address()
3595 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_loss_tmo_show()
3607 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_loss_tmo_store()
3628 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_reconnect_delay_show()
3638 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_reconnect_delay_store()
3655 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_fast_io_fail_tmo_show()
3665 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_fast_io_fail_tmo_store()
3690 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in cntrltype_show()
3707 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in dctype_show()
3720 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_dhchap_secret_show()
3731 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_dhchap_secret_store()
3770 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_dhchap_ctrl_secret_show()
3781 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_ctrl_dhchap_ctrl_secret_store()
3852 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);  in nvme_dev_attrs_are_visible()
3888 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,  in nvme_find_ns_head()
4001 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,  in nvme_alloc_ns_head()
4085 	struct nvme_ctrl *ctrl = ns->ctrl;  in nvme_init_ns_head()
4148 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)  in nvme_find_get_ns()
4184 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)  in nvme_alloc_ns()
4322 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)  in nvme_ns_remove_by_nsid()
4358 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)  in nvme_scan_ns()
4402 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,  in nvme_remove_invalid_namespaces()
4420 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)  in nvme_scan_ns_list()
4466 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)  in nvme_scan_ns_sequential()
4482 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)  in nvme_clear_changed_ns_log()
4509 	struct nvme_ctrl *ctrl =  in nvme_scan_work()
4510 		container_of(work, struct nvme_ctrl, scan_work);  in nvme_scan_work()
4547 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)  in nvme_remove_namespaces()
4585 	struct nvme_ctrl *ctrl =  in nvme_class_uevent()
4586 		container_of(dev, struct nvme_ctrl, ctrl_device);  in nvme_class_uevent()
4615 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)  in nvme_change_uevent()
4622 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)  in nvme_aen_uevent()
4640 	struct nvme_ctrl *ctrl =  in nvme_async_event_work()
4641 		container_of(work, struct nvme_ctrl, async_event_work);  in nvme_async_event_work()
4654 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)  in nvme_ctrl_pp_status()
4668 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)  in nvme_get_fw_slot_info()
4684 	struct nvme_ctrl *ctrl = container_of(work,  in nvme_fw_act_work()
4685 				struct nvme_ctrl, fw_act_work);  in nvme_fw_act_work()
4726 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)  in nvme_handle_aen_notice()
4766 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)  in nvme_handle_aer_persistent_error()
4773 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,  in nvme_complete_async_event()
4813 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,  in nvme_alloc_admin_tag_set()
4859 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)  in nvme_remove_admin_tag_set()
4868 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,  in nvme_alloc_io_tag_set()
4907 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)  in nvme_remove_io_tag_set()
4915 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)  in nvme_stop_ctrl()
4928 void nvme_start_ctrl(struct nvme_ctrl *ctrl)  in nvme_start_ctrl()
4954 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)  in nvme_uninit_ctrl()
4964 static void nvme_free_cels(struct nvme_ctrl *ctrl)  in nvme_free_cels()
4979 	struct nvme_ctrl *ctrl =  in nvme_free_ctrl()
4980 		container_of(dev, struct nvme_ctrl, ctrl_device);  in nvme_free_ctrl()
5010 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,  in nvme_init_ctrl()
5135 void nvme_kill_queues(struct nvme_ctrl *ctrl)  in nvme_kill_queues()
5152 void nvme_unfreeze(struct nvme_ctrl *ctrl)  in nvme_unfreeze()
5163 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)  in nvme_wait_freeze_timeout()
5178 void nvme_wait_freeze(struct nvme_ctrl *ctrl)  in nvme_wait_freeze()
5189 void nvme_start_freeze(struct nvme_ctrl *ctrl)  in nvme_start_freeze()
5200 void nvme_stop_queues(struct nvme_ctrl *ctrl)  in nvme_stop_queues()
5211 void nvme_start_queues(struct nvme_ctrl *ctrl)  in nvme_start_queues()
5222 void nvme_stop_admin_queue(struct nvme_ctrl *ctrl)  in nvme_stop_admin_queue()
5231 void nvme_start_admin_queue(struct nvme_ctrl *ctrl)  in nvme_start_admin_queue()
5238 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)  in nvme_sync_io_queues()
5249 void nvme_sync_queues(struct nvme_ctrl *ctrl)  in nvme_sync_queues()
5257 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)  in nvme_ctrl_from_file()