132 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)  in nvmet_async_events_free()  argument
137 		mutex_lock(&ctrl->lock);  in nvmet_async_events_free()
138 		if (!ctrl->nr_async_event_cmds) {  in nvmet_async_events_free()
139 			mutex_unlock(&ctrl->lock);  in nvmet_async_events_free()
143 		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];  in nvmet_async_events_free()
144 		mutex_unlock(&ctrl->lock);  in nvmet_async_events_free()
151 	struct nvmet_ctrl *ctrl =  in nvmet_async_event_work()  local
157 		mutex_lock(&ctrl->lock);  in nvmet_async_event_work()
158 		aen = list_first_entry_or_null(&ctrl->async_events,  in nvmet_async_event_work()
160 		if (!aen || !ctrl->nr_async_event_cmds) {  in nvmet_async_event_work()
161 			mutex_unlock(&ctrl->lock);  in nvmet_async_event_work()
165 		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];  in nvmet_async_event_work()
171 		mutex_unlock(&ctrl->lock);  in nvmet_async_event_work()
176 void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,  in nvmet_add_async_event()  argument
189 	mutex_lock(&ctrl->lock);  in nvmet_add_async_event()
190 	list_add_tail(&aen->entry, &ctrl->async_events);  in nvmet_add_async_event()
191 	mutex_unlock(&ctrl->lock);  in nvmet_add_async_event()
193 	schedule_work(&ctrl->async_event_work);  in nvmet_add_async_event()
196 static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid)  in nvmet_add_to_changed_ns_log()  argument
200 	mutex_lock(&ctrl->lock);  in nvmet_add_to_changed_ns_log()
201 	if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES)  in nvmet_add_to_changed_ns_log()
204 	for (i = 0; i < ctrl->nr_changed_ns; i++) {  in nvmet_add_to_changed_ns_log()
205 		if (ctrl->changed_ns_list[i] == nsid)  in nvmet_add_to_changed_ns_log()
209 	if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) {  in nvmet_add_to_changed_ns_log()
210 		ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff);  in nvmet_add_to_changed_ns_log()
211 		ctrl->nr_changed_ns = U32_MAX;  in nvmet_add_to_changed_ns_log()
215 	ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid;  in nvmet_add_to_changed_ns_log()
217 	mutex_unlock(&ctrl->lock);  in nvmet_add_to_changed_ns_log()
222 	struct nvmet_ctrl *ctrl;  in nvmet_ns_changed()  local
226 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_ns_changed()
227 		nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid));  in nvmet_ns_changed()
228 		if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_NS_ATTR))  in nvmet_ns_changed()
230 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,  in nvmet_ns_changed()
239 	struct nvmet_ctrl *ctrl;  in nvmet_send_ana_event()  local
242 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_send_ana_event()
243 		if (port && ctrl->port != port)  in nvmet_send_ana_event()
245 		if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_ANA_CHANGE))  in nvmet_send_ana_event()
247 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,  in nvmet_send_ana_event()
288 	struct nvmet_ctrl *ctrl;  in nvmet_port_del_ctrls()  local
291 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_port_del_ctrls()
292 		if (ctrl->port == port)  in nvmet_port_del_ctrls()
293 			ctrl->ops->delete_ctrl(ctrl);  in nvmet_port_del_ctrls()
352 	struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),  in nvmet_keep_alive_timer()  local
354 	bool cmd_seen = ctrl->cmd_seen;  in nvmet_keep_alive_timer()
356 	ctrl->cmd_seen = false;  in nvmet_keep_alive_timer()
359 			ctrl->cntlid);  in nvmet_keep_alive_timer()
360 		schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);  in nvmet_keep_alive_timer()
365 		ctrl->cntlid, ctrl->kato);  in nvmet_keep_alive_timer()
367 	nvmet_ctrl_fatal_error(ctrl);  in nvmet_keep_alive_timer()
370 static void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)  in nvmet_start_keep_alive_timer()  argument
373 		ctrl->cntlid, ctrl->kato);  in nvmet_start_keep_alive_timer()
375 	INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);  in nvmet_start_keep_alive_timer()
376 	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);  in nvmet_start_keep_alive_timer()
379 static void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)  in nvmet_stop_keep_alive_timer()  argument
381 	pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid);  in nvmet_stop_keep_alive_timer()
383 	cancel_delayed_work_sync(&ctrl->ka_work);  in nvmet_stop_keep_alive_timer()
386 static struct nvmet_ns *__nvmet_find_namespace(struct nvmet_ctrl *ctrl,  in __nvmet_find_namespace()  argument
391 	list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {  in __nvmet_find_namespace()
399 struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid)  in nvmet_find_namespace()  argument
404 	ns = __nvmet_find_namespace(ctrl, nsid);  in nvmet_find_namespace()
477 static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl,  in nvmet_p2pmem_ns_add_p2p()  argument
484 	if (!ctrl->p2p_client || !ns->use_p2pmem)  in nvmet_p2pmem_ns_add_p2p()
488 		ret = pci_p2pdma_distance(ns->p2p_dev, ctrl->p2p_client, true);  in nvmet_p2pmem_ns_add_p2p()
494 		clients[0] = ctrl->p2p_client;  in nvmet_p2pmem_ns_add_p2p()
500 			       dev_name(ctrl->p2p_client), ns->device_path);  in nvmet_p2pmem_ns_add_p2p()
505 	ret = radix_tree_insert(&ctrl->p2p_ns_map, ns->nsid, p2p_dev);  in nvmet_p2pmem_ns_add_p2p()
516 	struct nvmet_ctrl *ctrl;  in nvmet_ns_enable()  local
538 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvmet_ns_enable()
539 		nvmet_p2pmem_ns_add_p2p(ctrl, ns);  in nvmet_ns_enable()
575 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvmet_ns_enable()
576 		pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid));  in nvmet_ns_enable()
585 	struct nvmet_ctrl *ctrl;  in nvmet_ns_disable()  local
596 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvmet_ns_disable()
597 		pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid));  in nvmet_ns_disable()
676 	struct nvmet_ctrl *ctrl = req->sq->ctrl;  in nvmet_set_error()  local
682 	if (!ctrl || req->error_loc == NVMET_NO_ERROR_LOC)  in nvmet_set_error()
685 	spin_lock_irqsave(&ctrl->error_lock, flags);  in nvmet_set_error()
686 	ctrl->err_counter++;  in nvmet_set_error()
688 		&ctrl->slots[ctrl->err_counter % NVMET_ERROR_LOG_SLOTS];  in nvmet_set_error()
690 	new_error_slot->error_count = cpu_to_le64(ctrl->err_counter);  in nvmet_set_error()
697 	spin_unlock_irqrestore(&ctrl->error_lock, flags);  in nvmet_set_error()
727 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq,  in nvmet_cq_setup()  argument
733 	ctrl->cqs[qid] = cq;  in nvmet_cq_setup()
736 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,  in nvmet_sq_setup()  argument
743 	ctrl->sqs[qid] = sq;  in nvmet_sq_setup()
759 	if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq)  in nvmet_sq_destroy()
760 		nvmet_async_events_free(sq->ctrl);  in nvmet_sq_destroy()
766 	if (sq->ctrl) {  in nvmet_sq_destroy()
767 		nvmet_ctrl_put(sq->ctrl);  in nvmet_sq_destroy()
768 		sq->ctrl = NULL; /* allows reusing the queue later */  in nvmet_sq_destroy()
834 	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);  in nvmet_parse_io_cmd()
894 	if (unlikely(!req->sq->ctrl))  in nvmet_req_init()
901 	else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC)  in nvmet_req_init()
914 	if (sq->ctrl)  in nvmet_req_init()
915 		sq->ctrl->cmd_seen = true;  in nvmet_req_init()
948 		if (req->sq->ctrl && req->ns)  in nvmet_req_alloc_sgl()
949 			p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map,  in nvmet_req_alloc_sgl()
1023 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)  in nvmet_start_ctrl()  argument
1025 	lockdep_assert_held(&ctrl->lock);  in nvmet_start_ctrl()
1027 	if (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES ||  in nvmet_start_ctrl()
1028 	    nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES ||  in nvmet_start_ctrl()
1029 	    nvmet_cc_mps(ctrl->cc) != 0 ||  in nvmet_start_ctrl()
1030 	    nvmet_cc_ams(ctrl->cc) != 0 ||  in nvmet_start_ctrl()
1031 	    nvmet_cc_css(ctrl->cc) != 0) {  in nvmet_start_ctrl()
1032 		ctrl->csts = NVME_CSTS_CFS;  in nvmet_start_ctrl()
1036 	ctrl->csts = NVME_CSTS_RDY;  in nvmet_start_ctrl()
1044 	mod_delayed_work(system_wq, &ctrl->ka_work, ctrl->kato * HZ);  in nvmet_start_ctrl()
1047 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl)  in nvmet_clear_ctrl()  argument
1049 	lockdep_assert_held(&ctrl->lock);  in nvmet_clear_ctrl()
1052 	ctrl->csts &= ~NVME_CSTS_RDY;  in nvmet_clear_ctrl()
1053 	ctrl->cc = 0;  in nvmet_clear_ctrl()
1056 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new)  in nvmet_update_cc()  argument
1060 	mutex_lock(&ctrl->lock);  in nvmet_update_cc()
1061 	old = ctrl->cc;  in nvmet_update_cc()
1062 	ctrl->cc = new;  in nvmet_update_cc()
1065 		nvmet_start_ctrl(ctrl);  in nvmet_update_cc()
1067 		nvmet_clear_ctrl(ctrl);  in nvmet_update_cc()
1069 		nvmet_clear_ctrl(ctrl);  in nvmet_update_cc()
1070 		ctrl->csts |= NVME_CSTS_SHST_CMPLT;  in nvmet_update_cc()
1073 		ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;  in nvmet_update_cc()
1074 	mutex_unlock(&ctrl->lock);  in nvmet_update_cc()
1077 static void nvmet_init_cap(struct nvmet_ctrl *ctrl)  in nvmet_init_cap()  argument
1080 	ctrl->cap = (1ULL << 37);  in nvmet_init_cap()
1082 	ctrl->cap |= (15ULL << 24);  in nvmet_init_cap()
1084 	ctrl->cap |= NVMET_QUEUE_SIZE - 1;  in nvmet_init_cap()
1091 	struct nvmet_ctrl *ctrl;  in nvmet_ctrl_find_get()  local
1103 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {  in nvmet_ctrl_find_get()
1104 		if (ctrl->cntlid == cntlid) {  in nvmet_ctrl_find_get()
1105 			if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) {  in nvmet_ctrl_find_get()
1109 			if (!kref_get_unless_zero(&ctrl->ref))  in nvmet_ctrl_find_get()
1112 			*ret = ctrl;  in nvmet_ctrl_find_get()
1130 	if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {  in nvmet_check_ctrl_status()
1136 	if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {  in nvmet_check_ctrl_status()
1167 static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl,  in nvmet_setup_p2p_ns_map()  argument
1175 	ctrl->p2p_client = get_device(req->p2p_client);  in nvmet_setup_p2p_ns_map()
1177 	list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link)  in nvmet_setup_p2p_ns_map()
1178 		nvmet_p2pmem_ns_add_p2p(ctrl, ns);  in nvmet_setup_p2p_ns_map()
1184 static void nvmet_release_p2p_ns_map(struct nvmet_ctrl *ctrl)  in nvmet_release_p2p_ns_map()  argument
1189 	radix_tree_for_each_slot(slot, &ctrl->p2p_ns_map, &iter, 0)  in nvmet_release_p2p_ns_map()
1192 	put_device(ctrl->p2p_client);  in nvmet_release_p2p_ns_map()
1197 	struct nvmet_ctrl *ctrl =  in nvmet_fatal_error_handler()  local
1200 	pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid);  in nvmet_fatal_error_handler()
1201 	ctrl->ops->delete_ctrl(ctrl);  in nvmet_fatal_error_handler()
1208 	struct nvmet_ctrl *ctrl;  in nvmet_alloc_ctrl()  local
1234 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in nvmet_alloc_ctrl()
1235 	if (!ctrl)  in nvmet_alloc_ctrl()
1237 	mutex_init(&ctrl->lock);  in nvmet_alloc_ctrl()
1239 	nvmet_init_cap(ctrl);  in nvmet_alloc_ctrl()
1241 	ctrl->port = req->port;  in nvmet_alloc_ctrl()
1243 	INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);  in nvmet_alloc_ctrl()
1244 	INIT_LIST_HEAD(&ctrl->async_events);  in nvmet_alloc_ctrl()
1245 	INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL);  in nvmet_alloc_ctrl()
1246 	INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler);  in nvmet_alloc_ctrl()
1248 	memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);  in nvmet_alloc_ctrl()
1249 	memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);  in nvmet_alloc_ctrl()
1251 	kref_init(&ctrl->ref);  in nvmet_alloc_ctrl()
1252 	ctrl->subsys = subsys;  in nvmet_alloc_ctrl()
1253 	WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL);  in nvmet_alloc_ctrl()
1255 	ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES,  in nvmet_alloc_ctrl()
1257 	if (!ctrl->changed_ns_list)  in nvmet_alloc_ctrl()
1260 	ctrl->cqs = kcalloc(subsys->max_qid + 1,  in nvmet_alloc_ctrl()
1263 	if (!ctrl->cqs)  in nvmet_alloc_ctrl()
1266 	ctrl->sqs = kcalloc(subsys->max_qid + 1,  in nvmet_alloc_ctrl()
1269 	if (!ctrl->sqs)  in nvmet_alloc_ctrl()
1279 	ctrl->cntlid = ret;  in nvmet_alloc_ctrl()
1281 	ctrl->ops = req->ops;  in nvmet_alloc_ctrl()
1287 	if ((ctrl->subsys->type == NVME_NQN_DISC) && !kato)  in nvmet_alloc_ctrl()
1291 	ctrl->kato = DIV_ROUND_UP(kato, 1000);  in nvmet_alloc_ctrl()
1293 	ctrl->err_counter = 0;  in nvmet_alloc_ctrl()
1294 	spin_lock_init(&ctrl->error_lock);  in nvmet_alloc_ctrl()
1296 	nvmet_start_keep_alive_timer(ctrl);  in nvmet_alloc_ctrl()
1299 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);  in nvmet_alloc_ctrl()
1300 	nvmet_setup_p2p_ns_map(ctrl, req);  in nvmet_alloc_ctrl()
1303 	*ctrlp = ctrl;  in nvmet_alloc_ctrl()
1307 	kfree(ctrl->sqs);  in nvmet_alloc_ctrl()
1309 	kfree(ctrl->cqs);  in nvmet_alloc_ctrl()
1311 	kfree(ctrl->changed_ns_list);  in nvmet_alloc_ctrl()
1313 	kfree(ctrl);  in nvmet_alloc_ctrl()
1322 	struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref);  in nvmet_ctrl_free()  local
1323 	struct nvmet_subsys *subsys = ctrl->subsys;  in nvmet_ctrl_free()
1326 	nvmet_release_p2p_ns_map(ctrl);  in nvmet_ctrl_free()
1327 	list_del(&ctrl->subsys_entry);  in nvmet_ctrl_free()
1330 	nvmet_stop_keep_alive_timer(ctrl);  in nvmet_ctrl_free()
1332 	flush_work(&ctrl->async_event_work);  in nvmet_ctrl_free()
1333 	cancel_work_sync(&ctrl->fatal_err_work);  in nvmet_ctrl_free()
1335 	ida_simple_remove(&cntlid_ida, ctrl->cntlid);  in nvmet_ctrl_free()
1337 	kfree(ctrl->sqs);  in nvmet_ctrl_free()
1338 	kfree(ctrl->cqs);  in nvmet_ctrl_free()
1339 	kfree(ctrl->changed_ns_list);  in nvmet_ctrl_free()
1340 	kfree(ctrl);  in nvmet_ctrl_free()
1345 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl)  in nvmet_ctrl_put()  argument
1347 	kref_put(&ctrl->ref, nvmet_ctrl_free);  in nvmet_ctrl_put()
1350 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)  in nvmet_ctrl_fatal_error()  argument
1352 	mutex_lock(&ctrl->lock);  in nvmet_ctrl_fatal_error()
1353 	if (!(ctrl->csts & NVME_CSTS_CFS)) {  in nvmet_ctrl_fatal_error()
1354 		ctrl->csts |= NVME_CSTS_CFS;  in nvmet_ctrl_fatal_error()
1355 		schedule_work(&ctrl->fatal_err_work);  in nvmet_ctrl_fatal_error()
1357 	mutex_unlock(&ctrl->lock);  in nvmet_ctrl_fatal_error()
1445 	struct nvmet_ctrl *ctrl;  in nvmet_subsys_del_ctrls()  local
1448 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvmet_subsys_del_ctrls()
1449 		ctrl->ops->delete_ctrl(ctrl);  in nvmet_subsys_del_ctrls()