88 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)  in nvmet_async_events_free()  argument
93 		mutex_lock(&ctrl->lock);  in nvmet_async_events_free()
94 		if (!ctrl->nr_async_event_cmds) {  in nvmet_async_events_free()
95 			mutex_unlock(&ctrl->lock);  in nvmet_async_events_free()
99 		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];  in nvmet_async_events_free()
100 		mutex_unlock(&ctrl->lock);  in nvmet_async_events_free()
107 	struct nvmet_ctrl *ctrl =  in nvmet_async_event_work()  local
113 		mutex_lock(&ctrl->lock);  in nvmet_async_event_work()
114 		aen = list_first_entry_or_null(&ctrl->async_events,  in nvmet_async_event_work()
116 		if (!aen || !ctrl->nr_async_event_cmds) {  in nvmet_async_event_work()
117 			mutex_unlock(&ctrl->lock);  in nvmet_async_event_work()
121 		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];  in nvmet_async_event_work()
127 		mutex_unlock(&ctrl->lock);  in nvmet_async_event_work()
132 static void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,  in nvmet_add_async_event()  argument
145 	mutex_lock(&ctrl->lock);  in nvmet_add_async_event()
146 	list_add_tail(&aen->entry, &ctrl->async_events);  in nvmet_add_async_event()
147 	mutex_unlock(&ctrl->lock);  in nvmet_add_async_event()
149 	schedule_work(&ctrl->async_event_work);  in nvmet_add_async_event()
152 static bool nvmet_aen_disabled(struct nvmet_ctrl *ctrl, u32 aen)  in nvmet_aen_disabled()  argument
154 	if (!(READ_ONCE(ctrl->aen_enabled) & aen))  in nvmet_aen_disabled()
156 	return test_and_set_bit(aen, &ctrl->aen_masked);  in nvmet_aen_disabled()
159 static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid)  in nvmet_add_to_changed_ns_log()  argument
163 	mutex_lock(&ctrl->lock);  in nvmet_add_to_changed_ns_log()
164 	if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES)  in nvmet_add_to_changed_ns_log()
167 	for (i = 0; i < ctrl->nr_changed_ns; i++) {  in nvmet_add_to_changed_ns_log()
168 		if (ctrl->changed_ns_list[i] == nsid)  in nvmet_add_to_changed_ns_log()
172 	if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) {  in nvmet_add_to_changed_ns_log()
173 		ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff);  in nvmet_add_to_changed_ns_log()
174 		ctrl->nr_changed_ns = U32_MAX;  in nvmet_add_to_changed_ns_log()
178 	ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid;  in nvmet_add_to_changed_ns_log()
180 	mutex_unlock(&ctrl->lock);  in nvmet_add_to_changed_ns_log()
185 	struct nvmet_ctrl *ctrl;  in nvmet_ns_changed()  local
187 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_ns_changed()
188 		nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid));  in nvmet_ns_changed()
189 		if (nvmet_aen_disabled(ctrl, NVME_AEN_CFG_NS_ATTR))  in nvmet_ns_changed()
191 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,  in nvmet_ns_changed()
200 	struct nvmet_ctrl *ctrl;  in nvmet_send_ana_event()  local
203 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_send_ana_event()
204 		if (port && ctrl->port != port)  in nvmet_send_ana_event()
206 		if (nvmet_aen_disabled(ctrl, NVME_AEN_CFG_ANA_CHANGE))  in nvmet_send_ana_event()
208 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,  in nvmet_send_ana_event()
299 	struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),  in nvmet_keep_alive_timer()  local
303 		ctrl->cntlid, ctrl->kato);  in nvmet_keep_alive_timer()
305 	nvmet_ctrl_fatal_error(ctrl);  in nvmet_keep_alive_timer()
308 static void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)  in nvmet_start_keep_alive_timer()  argument
311 		ctrl->cntlid, ctrl->kato);  in nvmet_start_keep_alive_timer()
313 	INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);  in nvmet_start_keep_alive_timer()
314 	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);  in nvmet_start_keep_alive_timer()
317 static void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)  in nvmet_stop_keep_alive_timer()  argument
319 	pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid);  in nvmet_stop_keep_alive_timer()
321 	cancel_delayed_work_sync(&ctrl->ka_work);  in nvmet_stop_keep_alive_timer()
324 static struct nvmet_ns *__nvmet_find_namespace(struct nvmet_ctrl *ctrl,  in __nvmet_find_namespace()  argument
329 	list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {  in __nvmet_find_namespace()
337 struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid)  in nvmet_find_namespace()  argument
342 	ns = __nvmet_find_namespace(ctrl, nsid);  in nvmet_find_namespace()
529 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq,  in nvmet_cq_setup()  argument
535 	ctrl->cqs[qid] = cq;  in nvmet_cq_setup()
538 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,  in nvmet_sq_setup()  argument
545 	ctrl->sqs[qid] = sq;  in nvmet_sq_setup()
561 	if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq)  in nvmet_sq_destroy()
562 		nvmet_async_events_free(sq->ctrl);  in nvmet_sq_destroy()
568 	if (sq->ctrl) {  in nvmet_sq_destroy()
569 		nvmet_ctrl_put(sq->ctrl);  in nvmet_sq_destroy()
570 		sq->ctrl = NULL; /* allows reusing the queue later */  in nvmet_sq_destroy()
636 	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);  in nvmet_parse_io_cmd()
683 	if (unlikely(!req->sq->ctrl))  in nvmet_req_init()
690 	else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC)  in nvmet_req_init()
763 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)  in nvmet_start_ctrl()  argument
765 	lockdep_assert_held(&ctrl->lock);  in nvmet_start_ctrl()
767 	if (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES ||  in nvmet_start_ctrl()
768 	    nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES ||  in nvmet_start_ctrl()
769 	    nvmet_cc_mps(ctrl->cc) != 0 ||  in nvmet_start_ctrl()
770 	    nvmet_cc_ams(ctrl->cc) != 0 ||  in nvmet_start_ctrl()
771 	    nvmet_cc_css(ctrl->cc) != 0) {  in nvmet_start_ctrl()
772 		ctrl->csts = NVME_CSTS_CFS;  in nvmet_start_ctrl()
776 	ctrl->csts = NVME_CSTS_RDY;  in nvmet_start_ctrl()
784 	mod_delayed_work(system_wq, &ctrl->ka_work, ctrl->kato * HZ);  in nvmet_start_ctrl()
787 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl)  in nvmet_clear_ctrl()  argument
789 	lockdep_assert_held(&ctrl->lock);  in nvmet_clear_ctrl()
792 	ctrl->csts &= ~NVME_CSTS_RDY;  in nvmet_clear_ctrl()
793 	ctrl->cc = 0;  in nvmet_clear_ctrl()
796 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new)  in nvmet_update_cc()  argument
800 	mutex_lock(&ctrl->lock);  in nvmet_update_cc()
801 	old = ctrl->cc;  in nvmet_update_cc()
802 	ctrl->cc = new;  in nvmet_update_cc()
805 		nvmet_start_ctrl(ctrl);  in nvmet_update_cc()
807 		nvmet_clear_ctrl(ctrl);  in nvmet_update_cc()
809 		nvmet_clear_ctrl(ctrl);  in nvmet_update_cc()
810 		ctrl->csts |= NVME_CSTS_SHST_CMPLT;  in nvmet_update_cc()
813 		ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;  in nvmet_update_cc()
814 	mutex_unlock(&ctrl->lock);  in nvmet_update_cc()
817 static void nvmet_init_cap(struct nvmet_ctrl *ctrl)  in nvmet_init_cap()  argument
820 	ctrl->cap = (1ULL << 37);  in nvmet_init_cap()
822 	ctrl->cap |= (15ULL << 24);  in nvmet_init_cap()
824 	ctrl->cap |= NVMET_QUEUE_SIZE - 1;  in nvmet_init_cap()
831 	struct nvmet_ctrl *ctrl;  in nvmet_ctrl_find_get()  local
843 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_ctrl_find_get()
844 		if (ctrl->cntlid == cntlid) {  in nvmet_ctrl_find_get()
845 			if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) {  in nvmet_ctrl_find_get()
849 			if (!kref_get_unless_zero(&ctrl->ref))  in nvmet_ctrl_find_get()
852 			*ret = ctrl;  in nvmet_ctrl_find_get()
870 	if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {  in nvmet_check_ctrl_status()
876 	if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {  in nvmet_check_ctrl_status()
928 	struct nvmet_ctrl *ctrl;  in nvmet_alloc_ctrl()  local
954 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in nvmet_alloc_ctrl()
955 	if (!ctrl)  in nvmet_alloc_ctrl()
957 	mutex_init(&ctrl->lock);  in nvmet_alloc_ctrl()
959 	nvmet_init_cap(ctrl);  in nvmet_alloc_ctrl()
961 	ctrl->port = req->port;  in nvmet_alloc_ctrl()
963 	INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);  in nvmet_alloc_ctrl()
964 	INIT_LIST_HEAD(&ctrl->async_events);  in nvmet_alloc_ctrl()
966 	memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);  in nvmet_alloc_ctrl()
967 	memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);  in nvmet_alloc_ctrl()
969 	kref_init(&ctrl->ref);  in nvmet_alloc_ctrl()
970 	ctrl->subsys = subsys;  in nvmet_alloc_ctrl()
971 	WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL);  in nvmet_alloc_ctrl()
973 	ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES,  in nvmet_alloc_ctrl()
975 	if (!ctrl->changed_ns_list)  in nvmet_alloc_ctrl()
978 	ctrl->cqs = kcalloc(subsys->max_qid + 1,  in nvmet_alloc_ctrl()
981 	if (!ctrl->cqs)  in nvmet_alloc_ctrl()
984 	ctrl->sqs = kcalloc(subsys->max_qid + 1,  in nvmet_alloc_ctrl()
987 	if (!ctrl->sqs)  in nvmet_alloc_ctrl()
997 	ctrl->cntlid = ret;  in nvmet_alloc_ctrl()
999 	ctrl->ops = req->ops;  in nvmet_alloc_ctrl()
1000 	if (ctrl->subsys->type == NVME_NQN_DISC) {  in nvmet_alloc_ctrl()
1020 		ctrl->kato = NVMET_DISC_KATO;  in nvmet_alloc_ctrl()
1023 		ctrl->kato = DIV_ROUND_UP(kato, 1000);  in nvmet_alloc_ctrl()
1025 	nvmet_start_keep_alive_timer(ctrl);  in nvmet_alloc_ctrl()
1028 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);  in nvmet_alloc_ctrl()
1031 	*ctrlp = ctrl;  in nvmet_alloc_ctrl()
1035 	ida_simple_remove(&cntlid_ida, ctrl->cntlid);  in nvmet_alloc_ctrl()
1037 	kfree(ctrl->sqs);  in nvmet_alloc_ctrl()
1039 	kfree(ctrl->cqs);  in nvmet_alloc_ctrl()
1041 	kfree(ctrl->changed_ns_list);  in nvmet_alloc_ctrl()
1043 	kfree(ctrl);  in nvmet_alloc_ctrl()
1052 	struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref);  in nvmet_ctrl_free()  local
1053 	struct nvmet_subsys *subsys = ctrl->subsys;  in nvmet_ctrl_free()
1056 	list_del(&ctrl->subsys_entry);  in nvmet_ctrl_free()
1059 	nvmet_stop_keep_alive_timer(ctrl);  in nvmet_ctrl_free()
1061 	flush_work(&ctrl->async_event_work);  in nvmet_ctrl_free()
1062 	cancel_work_sync(&ctrl->fatal_err_work);  in nvmet_ctrl_free()
1064 	ida_simple_remove(&cntlid_ida, ctrl->cntlid);  in nvmet_ctrl_free()
1066 	kfree(ctrl->sqs);  in nvmet_ctrl_free()
1067 	kfree(ctrl->cqs);  in nvmet_ctrl_free()
1068 	kfree(ctrl->changed_ns_list);  in nvmet_ctrl_free()
1069 	kfree(ctrl);  in nvmet_ctrl_free()
1074 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl)  in nvmet_ctrl_put()  argument
1076 	kref_put(&ctrl->ref, nvmet_ctrl_free);  in nvmet_ctrl_put()
1081 	struct nvmet_ctrl *ctrl =  in nvmet_fatal_error_handler()  local
1084 	pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid);  in nvmet_fatal_error_handler()
1085 	ctrl->ops->delete_ctrl(ctrl);  in nvmet_fatal_error_handler()
1088 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)  in nvmet_ctrl_fatal_error()  argument
1090 	mutex_lock(&ctrl->lock);  in nvmet_ctrl_fatal_error()
1091 	if (!(ctrl->csts & NVME_CSTS_CFS)) {  in nvmet_ctrl_fatal_error()
1092 		ctrl->csts |= NVME_CSTS_CFS;  in nvmet_ctrl_fatal_error()
1093 		INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler);  in nvmet_ctrl_fatal_error()
1094 		schedule_work(&ctrl->fatal_err_work);  in nvmet_ctrl_fatal_error()
1096 	mutex_unlock(&ctrl->lock);  in nvmet_ctrl_fatal_error()
1185 	struct nvmet_ctrl *ctrl;  in nvmet_subsys_del_ctrls()  local
1188 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvmet_subsys_del_ctrls()
1189 		ctrl->ops->delete_ctrl(ctrl);  in nvmet_subsys_del_ctrls()