37 	struct nvme_ctrl	ctrl;  member
43 static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)  in to_loop_ctrl()  argument
45 	return container_of(ctrl, struct nvme_loop_ctrl, ctrl);  in to_loop_ctrl()
55 	struct nvme_loop_ctrl	*ctrl;  member
66 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
72 	return queue - queue->ctrl->queues;  in nvme_loop_queue_idx()
89 		return queue->ctrl->admin_tag_set.tags[queue_idx];  in nvme_loop_tagset()
90 	return queue->ctrl->tag_set.tags[queue_idx - 1];  in nvme_loop_tagset()
107 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,  in nvme_loop_queue_response()
114 			dev_err(queue->ctrl->ctrl.device,  in nvme_loop_queue_response()
142 	if (!nvmf_check_ready(&queue->ctrl->ctrl, req, queue_ready))  in nvme_loop_queue_rq()
143 		return nvmf_fail_nonready_command(&queue->ctrl->ctrl, req);  in nvme_loop_queue_rq()
151 	iod->req.port = queue->ctrl->port;  in nvme_loop_queue_rq()
176 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);  in nvme_loop_submit_async_event()  local
177 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_submit_async_event()
178 	struct nvme_loop_iod *iod = &ctrl->async_event_iod;  in nvme_loop_submit_async_event()
187 		dev_err(ctrl->ctrl.device, "failed async event work\n");  in nvme_loop_submit_async_event()
194 static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,  in nvme_loop_init_iod()  argument
199 	iod->queue = &ctrl->queues[queue_idx];  in nvme_loop_init_iod()
208 	struct nvme_loop_ctrl *ctrl = set->driver_data;  in nvme_loop_init_request()  local
210 	nvme_req(req)->ctrl = &ctrl->ctrl;  in nvme_loop_init_request()
211 	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),  in nvme_loop_init_request()
212 			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);  in nvme_loop_init_request()
218 	struct nvme_loop_ctrl *ctrl = data;  in nvme_loop_init_hctx()  local
219 	struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];  in nvme_loop_init_hctx()
221 	BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);  in nvme_loop_init_hctx()
230 	struct nvme_loop_ctrl *ctrl = data;  in nvme_loop_init_admin_hctx()  local
231 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_init_admin_hctx()
253 static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_admin_queue()  argument
255 	clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_destroy_admin_queue()
256 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_destroy_admin_queue()
257 	blk_cleanup_queue(ctrl->ctrl.admin_q);  in nvme_loop_destroy_admin_queue()
258 	blk_cleanup_queue(ctrl->ctrl.fabrics_q);  in nvme_loop_destroy_admin_queue()
259 	blk_mq_free_tag_set(&ctrl->admin_tag_set);  in nvme_loop_destroy_admin_queue()
264 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);  in nvme_loop_free_ctrl()  local
266 	if (list_empty(&ctrl->list))  in nvme_loop_free_ctrl()
270 	list_del(&ctrl->list);  in nvme_loop_free_ctrl()
274 		blk_cleanup_queue(ctrl->ctrl.connect_q);  in nvme_loop_free_ctrl()
275 		blk_mq_free_tag_set(&ctrl->tag_set);  in nvme_loop_free_ctrl()
277 	kfree(ctrl->queues);  in nvme_loop_free_ctrl()
280 	kfree(ctrl);  in nvme_loop_free_ctrl()
283 static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_io_queues()  argument
287 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_destroy_io_queues()
288 		clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_destroy_io_queues()
289 		nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);  in nvme_loop_destroy_io_queues()
293 static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_init_io_queues()  argument
295 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;  in nvme_loop_init_io_queues()
300 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);  in nvme_loop_init_io_queues()
304 	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);  in nvme_loop_init_io_queues()
307 		ctrl->queues[i].ctrl = ctrl;  in nvme_loop_init_io_queues()
308 		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);  in nvme_loop_init_io_queues()
312 		ctrl->ctrl.queue_count++;  in nvme_loop_init_io_queues()
318 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_init_io_queues()
322 static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_connect_io_queues()  argument
326 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_connect_io_queues()
327 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i, false);  in nvme_loop_connect_io_queues()
330 		set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_connect_io_queues()
336 static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_configure_admin_queue()  argument
340 	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));  in nvme_loop_configure_admin_queue()
341 	ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;  in nvme_loop_configure_admin_queue()
342 	ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;  in nvme_loop_configure_admin_queue()
343 	ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */  in nvme_loop_configure_admin_queue()
344 	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;  in nvme_loop_configure_admin_queue()
345 	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +  in nvme_loop_configure_admin_queue()
347 	ctrl->admin_tag_set.driver_data = ctrl;  in nvme_loop_configure_admin_queue()
348 	ctrl->admin_tag_set.nr_hw_queues = 1;  in nvme_loop_configure_admin_queue()
349 	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;  in nvme_loop_configure_admin_queue()
350 	ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;  in nvme_loop_configure_admin_queue()
352 	ctrl->queues[0].ctrl = ctrl;  in nvme_loop_configure_admin_queue()
353 	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
356 	ctrl->ctrl.queue_count = 1;  in nvme_loop_configure_admin_queue()
358 	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
361 	ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;  in nvme_loop_configure_admin_queue()
363 	ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
364 	if (IS_ERR(ctrl->ctrl.fabrics_q)) {  in nvme_loop_configure_admin_queue()
365 		error = PTR_ERR(ctrl->ctrl.fabrics_q);  in nvme_loop_configure_admin_queue()
369 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
370 	if (IS_ERR(ctrl->ctrl.admin_q)) {  in nvme_loop_configure_admin_queue()
371 		error = PTR_ERR(ctrl->ctrl.admin_q);  in nvme_loop_configure_admin_queue()
375 	error = nvmf_connect_admin_queue(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
379 	set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_configure_admin_queue()
381 	error = nvme_enable_ctrl(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
385 	ctrl->ctrl.max_hw_sectors =  in nvme_loop_configure_admin_queue()
388 	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);  in nvme_loop_configure_admin_queue()
390 	error = nvme_init_identify(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
397 	blk_cleanup_queue(ctrl->ctrl.admin_q);  in nvme_loop_configure_admin_queue()
399 	blk_cleanup_queue(ctrl->ctrl.fabrics_q);  in nvme_loop_configure_admin_queue()
401 	blk_mq_free_tag_set(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
403 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
407 static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)  in nvme_loop_shutdown_ctrl()  argument
409 	if (ctrl->ctrl.queue_count > 1) {  in nvme_loop_shutdown_ctrl()
410 		nvme_stop_queues(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
411 		blk_mq_tagset_busy_iter(&ctrl->tag_set,  in nvme_loop_shutdown_ctrl()
412 					nvme_cancel_request, &ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
413 		blk_mq_tagset_wait_completed_request(&ctrl->tag_set);  in nvme_loop_shutdown_ctrl()
414 		nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_shutdown_ctrl()
417 	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);  in nvme_loop_shutdown_ctrl()
418 	if (ctrl->ctrl.state == NVME_CTRL_LIVE)  in nvme_loop_shutdown_ctrl()
419 		nvme_shutdown_ctrl(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
421 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,  in nvme_loop_shutdown_ctrl()
422 				nvme_cancel_request, &ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
423 	blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);  in nvme_loop_shutdown_ctrl()
424 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_shutdown_ctrl()
427 static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)  in nvme_loop_delete_ctrl_host()  argument
429 	nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));  in nvme_loop_delete_ctrl_host()
434 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_delete_ctrl()  local
437 	list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {  in nvme_loop_delete_ctrl()
438 		if (ctrl->ctrl.cntlid == nctrl->cntlid)  in nvme_loop_delete_ctrl()
439 			nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_delete_ctrl()
446 	struct nvme_loop_ctrl *ctrl =  in nvme_loop_reset_ctrl_work()  local
447 		container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);  in nvme_loop_reset_ctrl_work()
451 	nvme_stop_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
452 	nvme_loop_shutdown_ctrl(ctrl);  in nvme_loop_reset_ctrl_work()
454 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {  in nvme_loop_reset_ctrl_work()
460 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
464 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
468 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
472 	blk_mq_update_nr_hw_queues(&ctrl->tag_set,  in nvme_loop_reset_ctrl_work()
473 			ctrl->ctrl.queue_count - 1);  in nvme_loop_reset_ctrl_work()
475 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);  in nvme_loop_reset_ctrl_work()
478 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
483 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
485 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
487 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");  in nvme_loop_reset_ctrl_work()
488 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
489 	nvme_put_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
505 static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_create_io_queues()  argument
509 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_create_io_queues()
513 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));  in nvme_loop_create_io_queues()
514 	ctrl->tag_set.ops = &nvme_loop_mq_ops;  in nvme_loop_create_io_queues()
515 	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;  in nvme_loop_create_io_queues()
516 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */  in nvme_loop_create_io_queues()
517 	ctrl->tag_set.numa_node = NUMA_NO_NODE;  in nvme_loop_create_io_queues()
518 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in nvme_loop_create_io_queues()
519 	ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +  in nvme_loop_create_io_queues()
521 	ctrl->tag_set.driver_data = ctrl;  in nvme_loop_create_io_queues()
522 	ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;  in nvme_loop_create_io_queues()
523 	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;  in nvme_loop_create_io_queues()
524 	ctrl->ctrl.tagset = &ctrl->tag_set;  in nvme_loop_create_io_queues()
526 	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);  in nvme_loop_create_io_queues()
530 	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);  in nvme_loop_create_io_queues()
531 	if (IS_ERR(ctrl->ctrl.connect_q)) {  in nvme_loop_create_io_queues()
532 		ret = PTR_ERR(ctrl->ctrl.connect_q);  in nvme_loop_create_io_queues()
536 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_create_io_queues()
543 	blk_cleanup_queue(ctrl->ctrl.connect_q);  in nvme_loop_create_io_queues()
545 	blk_mq_free_tag_set(&ctrl->tag_set);  in nvme_loop_create_io_queues()
547 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_create_io_queues()
551 static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)  in nvme_loop_find_port()  argument
558 		if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&  in nvme_loop_find_port()
559 		    strcmp(ctrl->opts->traddr, p->disc_addr.traddr))  in nvme_loop_find_port()
571 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_create_ctrl()  local
575 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in nvme_loop_create_ctrl()
576 	if (!ctrl)  in nvme_loop_create_ctrl()
578 	ctrl->ctrl.opts = opts;  in nvme_loop_create_ctrl()
579 	INIT_LIST_HEAD(&ctrl->list);  in nvme_loop_create_ctrl()
581 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);  in nvme_loop_create_ctrl()
583 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,  in nvme_loop_create_ctrl()
590 	ctrl->ctrl.sqsize = opts->queue_size - 1;  in nvme_loop_create_ctrl()
591 	ctrl->ctrl.kato = opts->kato;  in nvme_loop_create_ctrl()
592 	ctrl->port = nvme_loop_find_port(&ctrl->ctrl);  in nvme_loop_create_ctrl()
594 	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),  in nvme_loop_create_ctrl()
596 	if (!ctrl->queues)  in nvme_loop_create_ctrl()
599 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_create_ctrl()
603 	if (opts->queue_size > ctrl->ctrl.maxcmd) {  in nvme_loop_create_ctrl()
605 		dev_warn(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
607 			opts->queue_size, ctrl->ctrl.maxcmd);  in nvme_loop_create_ctrl()
608 		opts->queue_size = ctrl->ctrl.maxcmd;  in nvme_loop_create_ctrl()
612 		ret = nvme_loop_create_io_queues(ctrl);  in nvme_loop_create_ctrl()
617 	nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);  in nvme_loop_create_ctrl()
619 	dev_info(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
620 		 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);  in nvme_loop_create_ctrl()
622 	nvme_get_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
624 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);  in nvme_loop_create_ctrl()
628 	list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);  in nvme_loop_create_ctrl()
631 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
633 	return &ctrl->ctrl;  in nvme_loop_create_ctrl()
636 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_create_ctrl()
638 	kfree(ctrl->queues);  in nvme_loop_create_ctrl()
640 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
642 	nvme_put_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
704 	struct nvme_loop_ctrl *ctrl, *next;  in nvme_loop_cleanup_module()  local
710 	list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)  in nvme_loop_cleanup_module()
711 		nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_cleanup_module()