Lines Matching refs:ctrl
83 struct nvme_tcp_ctrl *ctrl; member
111 struct nvme_ctrl ctrl; member
125 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) in to_tcp_ctrl() argument
127 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); in to_tcp_ctrl()
132 return queue - queue->ctrl->queues; in nvme_tcp_queue_id()
140 return queue->ctrl->admin_tag_set.tags[queue_idx]; in nvme_tcp_tagset()
141 return queue->ctrl->tag_set.tags[queue_idx - 1]; in nvme_tcp_tagset()
161 return req == &req->queue->ctrl->async_req; in nvme_tcp_async_req()
314 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_verify_hdgst()
324 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_verify_hdgst()
343 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_check_ddgst()
365 struct nvme_tcp_ctrl *ctrl = set->driver_data; in nvme_tcp_init_request() local
367 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; in nvme_tcp_init_request()
368 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; in nvme_tcp_init_request()
378 nvme_req(rq)->ctrl = &ctrl->ctrl; in nvme_tcp_init_request()
386 struct nvme_tcp_ctrl *ctrl = data; in nvme_tcp_init_hctx() local
387 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; in nvme_tcp_init_hctx()
396 struct nvme_tcp_ctrl *ctrl = data; in nvme_tcp_init_admin_hctx() local
397 struct nvme_tcp_queue *queue = &ctrl->queues[0]; in nvme_tcp_init_admin_hctx()
420 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) in nvme_tcp_error_recovery() argument
422 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) in nvme_tcp_error_recovery()
425 queue_work(nvme_wq, &to_tcp_ctrl(ctrl)->err_work); in nvme_tcp_error_recovery()
435 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_process_nvme_cqe()
438 nvme_tcp_error_recovery(&queue->ctrl->ctrl); in nvme_tcp_process_nvme_cqe()
455 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_handle_c2h_data()
462 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_handle_c2h_data()
472 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_handle_c2h_data()
475 nvme_tcp_error_recovery(&queue->ctrl->ctrl); in nvme_tcp_handle_c2h_data()
496 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, in nvme_tcp_handle_comp()
517 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_setup_h2c_data_pdu()
525 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_setup_h2c_data_pdu()
559 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_handle_r2t()
622 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_pdu()
644 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_data()
666 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_data()
686 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_data()
733 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_ddgst()
774 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_recv_skb()
777 nvme_tcp_error_recovery(&queue->ctrl->ctrl); in nvme_tcp_recv_skb()
825 nvme_tcp_error_recovery(&queue->ctrl->ctrl); in nvme_tcp_state_change()
828 dev_info(queue->ctrl->ctrl.device, in nvme_tcp_state_change()
1055 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_io_work()
1110 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) in nvme_tcp_free_async_req() argument
1112 struct nvme_tcp_request *async = &ctrl->async_req; in nvme_tcp_free_async_req()
1117 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) in nvme_tcp_alloc_async_req() argument
1119 struct nvme_tcp_queue *queue = &ctrl->queues[0]; in nvme_tcp_alloc_async_req()
1120 struct nvme_tcp_request *async = &ctrl->async_req; in nvme_tcp_alloc_async_req()
1129 async->queue = &ctrl->queues[0]; in nvme_tcp_alloc_async_req()
1135 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_free_queue() local
1136 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; in nvme_tcp_free_queue()
1249 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_alloc_queue() local
1250 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; in nvme_tcp_alloc_queue()
1254 queue->ctrl = ctrl; in nvme_tcp_alloc_queue()
1266 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, in nvme_tcp_alloc_queue()
1333 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, in nvme_tcp_alloc_queue()
1334 sizeof(ctrl->src_addr)); in nvme_tcp_alloc_queue()
1365 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, in nvme_tcp_alloc_queue()
1366 sizeof(ctrl->addr), 0); in nvme_tcp_alloc_queue()
1430 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_stop_queue() local
1431 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; in nvme_tcp_stop_queue()
1441 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_start_queue() local
1450 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags); in nvme_tcp_start_queue()
1452 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags)) in nvme_tcp_start_queue()
1453 __nvme_tcp_stop_queue(&ctrl->queues[idx]); in nvme_tcp_start_queue()
1463 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_alloc_tagset() local
1468 set = &ctrl->admin_tag_set; in nvme_tcp_alloc_tagset()
1475 set->driver_data = ctrl; in nvme_tcp_alloc_tagset()
1479 set = &ctrl->tag_set; in nvme_tcp_alloc_tagset()
1487 set->driver_data = ctrl; in nvme_tcp_alloc_tagset()
1500 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) in nvme_tcp_free_admin_queue() argument
1502 if (to_tcp_ctrl(ctrl)->async_req.pdu) { in nvme_tcp_free_admin_queue()
1503 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); in nvme_tcp_free_admin_queue()
1504 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; in nvme_tcp_free_admin_queue()
1507 nvme_tcp_free_queue(ctrl, 0); in nvme_tcp_free_admin_queue()
1510 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) in nvme_tcp_free_io_queues() argument
1514 for (i = 1; i < ctrl->queue_count; i++) in nvme_tcp_free_io_queues()
1515 nvme_tcp_free_queue(ctrl, i); in nvme_tcp_free_io_queues()
1518 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) in nvme_tcp_stop_io_queues() argument
1522 for (i = 1; i < ctrl->queue_count; i++) in nvme_tcp_stop_io_queues()
1523 nvme_tcp_stop_queue(ctrl, i); in nvme_tcp_stop_io_queues()
1526 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) in nvme_tcp_start_io_queues() argument
1530 for (i = 1; i < ctrl->queue_count; i++) { in nvme_tcp_start_io_queues()
1531 ret = nvme_tcp_start_queue(ctrl, i); in nvme_tcp_start_io_queues()
1540 nvme_tcp_stop_queue(ctrl, i); in nvme_tcp_start_io_queues()
1544 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) in nvme_tcp_alloc_admin_queue() argument
1548 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); in nvme_tcp_alloc_admin_queue()
1552 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); in nvme_tcp_alloc_admin_queue()
1559 nvme_tcp_free_queue(ctrl, 0); in nvme_tcp_alloc_admin_queue()
1563 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) in __nvme_tcp_alloc_io_queues() argument
1567 for (i = 1; i < ctrl->queue_count; i++) { in __nvme_tcp_alloc_io_queues()
1568 ret = nvme_tcp_alloc_queue(ctrl, i, in __nvme_tcp_alloc_io_queues()
1569 ctrl->sqsize + 1); in __nvme_tcp_alloc_io_queues()
1578 nvme_tcp_free_queue(ctrl, i); in __nvme_tcp_alloc_io_queues()
1583 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) in nvme_tcp_nr_io_queues() argument
1587 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); in nvme_tcp_nr_io_queues()
1588 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); in nvme_tcp_nr_io_queues()
1589 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); in nvme_tcp_nr_io_queues()
1597 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_set_io_queues() local
1606 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; in nvme_tcp_set_io_queues()
1607 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; in nvme_tcp_set_io_queues()
1608 ctrl->io_queues[HCTX_TYPE_DEFAULT] = in nvme_tcp_set_io_queues()
1610 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_set_io_queues()
1617 ctrl->io_queues[HCTX_TYPE_DEFAULT] = in nvme_tcp_set_io_queues()
1619 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_set_io_queues()
1624 ctrl->io_queues[HCTX_TYPE_POLL] = in nvme_tcp_set_io_queues()
1629 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) in nvme_tcp_alloc_io_queues() argument
1634 nr_io_queues = nvme_tcp_nr_io_queues(ctrl); in nvme_tcp_alloc_io_queues()
1635 ret = nvme_set_queue_count(ctrl, &nr_io_queues); in nvme_tcp_alloc_io_queues()
1639 ctrl->queue_count = nr_io_queues + 1; in nvme_tcp_alloc_io_queues()
1640 if (ctrl->queue_count < 2) in nvme_tcp_alloc_io_queues()
1643 dev_info(ctrl->device, in nvme_tcp_alloc_io_queues()
1646 nvme_tcp_set_io_queues(ctrl, nr_io_queues); in nvme_tcp_alloc_io_queues()
1648 return __nvme_tcp_alloc_io_queues(ctrl); in nvme_tcp_alloc_io_queues()
1651 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) in nvme_tcp_destroy_io_queues() argument
1653 nvme_tcp_stop_io_queues(ctrl); in nvme_tcp_destroy_io_queues()
1655 blk_cleanup_queue(ctrl->connect_q); in nvme_tcp_destroy_io_queues()
1656 blk_mq_free_tag_set(ctrl->tagset); in nvme_tcp_destroy_io_queues()
1658 nvme_tcp_free_io_queues(ctrl); in nvme_tcp_destroy_io_queues()
1661 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) in nvme_tcp_configure_io_queues() argument
1665 ret = nvme_tcp_alloc_io_queues(ctrl); in nvme_tcp_configure_io_queues()
1670 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false); in nvme_tcp_configure_io_queues()
1671 if (IS_ERR(ctrl->tagset)) { in nvme_tcp_configure_io_queues()
1672 ret = PTR_ERR(ctrl->tagset); in nvme_tcp_configure_io_queues()
1676 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset); in nvme_tcp_configure_io_queues()
1677 if (IS_ERR(ctrl->connect_q)) { in nvme_tcp_configure_io_queues()
1678 ret = PTR_ERR(ctrl->connect_q); in nvme_tcp_configure_io_queues()
1682 blk_mq_update_nr_hw_queues(ctrl->tagset, in nvme_tcp_configure_io_queues()
1683 ctrl->queue_count - 1); in nvme_tcp_configure_io_queues()
1686 ret = nvme_tcp_start_io_queues(ctrl); in nvme_tcp_configure_io_queues()
1694 blk_cleanup_queue(ctrl->connect_q); in nvme_tcp_configure_io_queues()
1697 blk_mq_free_tag_set(ctrl->tagset); in nvme_tcp_configure_io_queues()
1699 nvme_tcp_free_io_queues(ctrl); in nvme_tcp_configure_io_queues()
1703 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) in nvme_tcp_destroy_admin_queue() argument
1705 nvme_tcp_stop_queue(ctrl, 0); in nvme_tcp_destroy_admin_queue()
1707 blk_cleanup_queue(ctrl->admin_q); in nvme_tcp_destroy_admin_queue()
1708 blk_cleanup_queue(ctrl->fabrics_q); in nvme_tcp_destroy_admin_queue()
1709 blk_mq_free_tag_set(ctrl->admin_tagset); in nvme_tcp_destroy_admin_queue()
1711 nvme_tcp_free_admin_queue(ctrl); in nvme_tcp_destroy_admin_queue()
1714 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) in nvme_tcp_configure_admin_queue() argument
1718 error = nvme_tcp_alloc_admin_queue(ctrl); in nvme_tcp_configure_admin_queue()
1723 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true); in nvme_tcp_configure_admin_queue()
1724 if (IS_ERR(ctrl->admin_tagset)) { in nvme_tcp_configure_admin_queue()
1725 error = PTR_ERR(ctrl->admin_tagset); in nvme_tcp_configure_admin_queue()
1729 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); in nvme_tcp_configure_admin_queue()
1730 if (IS_ERR(ctrl->fabrics_q)) { in nvme_tcp_configure_admin_queue()
1731 error = PTR_ERR(ctrl->fabrics_q); in nvme_tcp_configure_admin_queue()
1735 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); in nvme_tcp_configure_admin_queue()
1736 if (IS_ERR(ctrl->admin_q)) { in nvme_tcp_configure_admin_queue()
1737 error = PTR_ERR(ctrl->admin_q); in nvme_tcp_configure_admin_queue()
1742 error = nvme_tcp_start_queue(ctrl, 0); in nvme_tcp_configure_admin_queue()
1746 error = nvme_enable_ctrl(ctrl); in nvme_tcp_configure_admin_queue()
1750 blk_mq_unquiesce_queue(ctrl->admin_q); in nvme_tcp_configure_admin_queue()
1752 error = nvme_init_identify(ctrl); in nvme_tcp_configure_admin_queue()
1759 nvme_tcp_stop_queue(ctrl, 0); in nvme_tcp_configure_admin_queue()
1762 blk_cleanup_queue(ctrl->admin_q); in nvme_tcp_configure_admin_queue()
1765 blk_cleanup_queue(ctrl->fabrics_q); in nvme_tcp_configure_admin_queue()
1768 blk_mq_free_tag_set(ctrl->admin_tagset); in nvme_tcp_configure_admin_queue()
1770 nvme_tcp_free_admin_queue(ctrl); in nvme_tcp_configure_admin_queue()
1774 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, in nvme_tcp_teardown_admin_queue() argument
1777 blk_mq_quiesce_queue(ctrl->admin_q); in nvme_tcp_teardown_admin_queue()
1778 nvme_tcp_stop_queue(ctrl, 0); in nvme_tcp_teardown_admin_queue()
1779 if (ctrl->admin_tagset) { in nvme_tcp_teardown_admin_queue()
1780 blk_mq_tagset_busy_iter(ctrl->admin_tagset, in nvme_tcp_teardown_admin_queue()
1781 nvme_cancel_request, ctrl); in nvme_tcp_teardown_admin_queue()
1782 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset); in nvme_tcp_teardown_admin_queue()
1785 blk_mq_unquiesce_queue(ctrl->admin_q); in nvme_tcp_teardown_admin_queue()
1786 nvme_tcp_destroy_admin_queue(ctrl, remove); in nvme_tcp_teardown_admin_queue()
1789 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, in nvme_tcp_teardown_io_queues() argument
1792 if (ctrl->queue_count <= 1) in nvme_tcp_teardown_io_queues()
1794 nvme_stop_queues(ctrl); in nvme_tcp_teardown_io_queues()
1795 nvme_tcp_stop_io_queues(ctrl); in nvme_tcp_teardown_io_queues()
1796 if (ctrl->tagset) { in nvme_tcp_teardown_io_queues()
1797 blk_mq_tagset_busy_iter(ctrl->tagset, in nvme_tcp_teardown_io_queues()
1798 nvme_cancel_request, ctrl); in nvme_tcp_teardown_io_queues()
1799 blk_mq_tagset_wait_completed_request(ctrl->tagset); in nvme_tcp_teardown_io_queues()
1802 nvme_start_queues(ctrl); in nvme_tcp_teardown_io_queues()
1803 nvme_tcp_destroy_io_queues(ctrl, remove); in nvme_tcp_teardown_io_queues()
1806 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) in nvme_tcp_reconnect_or_remove() argument
1809 if (ctrl->state != NVME_CTRL_CONNECTING) { in nvme_tcp_reconnect_or_remove()
1810 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || in nvme_tcp_reconnect_or_remove()
1811 ctrl->state == NVME_CTRL_LIVE); in nvme_tcp_reconnect_or_remove()
1815 if (nvmf_should_reconnect(ctrl)) { in nvme_tcp_reconnect_or_remove()
1816 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", in nvme_tcp_reconnect_or_remove()
1817 ctrl->opts->reconnect_delay); in nvme_tcp_reconnect_or_remove()
1818 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, in nvme_tcp_reconnect_or_remove()
1819 ctrl->opts->reconnect_delay * HZ); in nvme_tcp_reconnect_or_remove()
1821 dev_info(ctrl->device, "Removing controller...\n"); in nvme_tcp_reconnect_or_remove()
1822 nvme_delete_ctrl(ctrl); in nvme_tcp_reconnect_or_remove()
1826 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) in nvme_tcp_setup_ctrl() argument
1828 struct nvmf_ctrl_options *opts = ctrl->opts; in nvme_tcp_setup_ctrl()
1831 ret = nvme_tcp_configure_admin_queue(ctrl, new); in nvme_tcp_setup_ctrl()
1835 if (ctrl->icdoff) { in nvme_tcp_setup_ctrl()
1836 dev_err(ctrl->device, "icdoff is not supported!\n"); in nvme_tcp_setup_ctrl()
1840 if (opts->queue_size > ctrl->sqsize + 1) in nvme_tcp_setup_ctrl()
1841 dev_warn(ctrl->device, in nvme_tcp_setup_ctrl()
1843 opts->queue_size, ctrl->sqsize + 1); in nvme_tcp_setup_ctrl()
1845 if (ctrl->sqsize + 1 > ctrl->maxcmd) { in nvme_tcp_setup_ctrl()
1846 dev_warn(ctrl->device, in nvme_tcp_setup_ctrl()
1848 ctrl->sqsize + 1, ctrl->maxcmd); in nvme_tcp_setup_ctrl()
1849 ctrl->sqsize = ctrl->maxcmd - 1; in nvme_tcp_setup_ctrl()
1852 if (ctrl->queue_count > 1) { in nvme_tcp_setup_ctrl()
1853 ret = nvme_tcp_configure_io_queues(ctrl, new); in nvme_tcp_setup_ctrl()
1858 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { in nvme_tcp_setup_ctrl()
1860 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); in nvme_tcp_setup_ctrl()
1865 nvme_start_ctrl(ctrl); in nvme_tcp_setup_ctrl()
1869 if (ctrl->queue_count > 1) in nvme_tcp_setup_ctrl()
1870 nvme_tcp_destroy_io_queues(ctrl, new); in nvme_tcp_setup_ctrl()
1872 nvme_tcp_stop_queue(ctrl, 0); in nvme_tcp_setup_ctrl()
1873 nvme_tcp_destroy_admin_queue(ctrl, new); in nvme_tcp_setup_ctrl()
1881 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; in nvme_tcp_reconnect_ctrl_work() local
1883 ++ctrl->nr_reconnects; in nvme_tcp_reconnect_ctrl_work()
1885 if (nvme_tcp_setup_ctrl(ctrl, false)) in nvme_tcp_reconnect_ctrl_work()
1888 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", in nvme_tcp_reconnect_ctrl_work()
1889 ctrl->nr_reconnects); in nvme_tcp_reconnect_ctrl_work()
1891 ctrl->nr_reconnects = 0; in nvme_tcp_reconnect_ctrl_work()
1896 dev_info(ctrl->device, "Failed reconnect attempt %d\n", in nvme_tcp_reconnect_ctrl_work()
1897 ctrl->nr_reconnects); in nvme_tcp_reconnect_ctrl_work()
1898 nvme_tcp_reconnect_or_remove(ctrl); in nvme_tcp_reconnect_ctrl_work()
1905 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; in nvme_tcp_error_recovery_work() local
1907 nvme_stop_keep_alive(ctrl); in nvme_tcp_error_recovery_work()
1908 nvme_tcp_teardown_io_queues(ctrl, false); in nvme_tcp_error_recovery_work()
1910 nvme_start_queues(ctrl); in nvme_tcp_error_recovery_work()
1911 nvme_tcp_teardown_admin_queue(ctrl, false); in nvme_tcp_error_recovery_work()
1912 blk_mq_unquiesce_queue(ctrl->admin_q); in nvme_tcp_error_recovery_work()
1914 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { in nvme_tcp_error_recovery_work()
1916 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); in nvme_tcp_error_recovery_work()
1920 nvme_tcp_reconnect_or_remove(ctrl); in nvme_tcp_error_recovery_work()
1923 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) in nvme_tcp_teardown_ctrl() argument
1925 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); in nvme_tcp_teardown_ctrl()
1926 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); in nvme_tcp_teardown_ctrl()
1928 nvme_tcp_teardown_io_queues(ctrl, shutdown); in nvme_tcp_teardown_ctrl()
1929 blk_mq_quiesce_queue(ctrl->admin_q); in nvme_tcp_teardown_ctrl()
1931 nvme_shutdown_ctrl(ctrl); in nvme_tcp_teardown_ctrl()
1933 nvme_disable_ctrl(ctrl); in nvme_tcp_teardown_ctrl()
1934 nvme_tcp_teardown_admin_queue(ctrl, shutdown); in nvme_tcp_teardown_ctrl()
1937 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) in nvme_tcp_delete_ctrl() argument
1939 nvme_tcp_teardown_ctrl(ctrl, true); in nvme_tcp_delete_ctrl()
1944 struct nvme_ctrl *ctrl = in nvme_reset_ctrl_work() local
1947 nvme_stop_ctrl(ctrl); in nvme_reset_ctrl_work()
1948 nvme_tcp_teardown_ctrl(ctrl, false); in nvme_reset_ctrl_work()
1950 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { in nvme_reset_ctrl_work()
1952 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); in nvme_reset_ctrl_work()
1956 if (nvme_tcp_setup_ctrl(ctrl, false)) in nvme_reset_ctrl_work()
1962 ++ctrl->nr_reconnects; in nvme_reset_ctrl_work()
1963 nvme_tcp_reconnect_or_remove(ctrl); in nvme_reset_ctrl_work()
1968 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); in nvme_tcp_free_ctrl() local
1970 if (list_empty(&ctrl->list)) in nvme_tcp_free_ctrl()
1974 list_del(&ctrl->list); in nvme_tcp_free_ctrl()
1979 kfree(ctrl->queues); in nvme_tcp_free_ctrl()
1980 kfree(ctrl); in nvme_tcp_free_ctrl()
1998 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); in nvme_tcp_set_sg_inline()
2016 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); in nvme_tcp_submit_async_event() local
2017 struct nvme_tcp_queue *queue = &ctrl->queues[0]; in nvme_tcp_submit_async_event()
2018 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; in nvme_tcp_submit_async_event()
2034 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; in nvme_tcp_submit_async_event()
2035 ctrl->async_req.offset = 0; in nvme_tcp_submit_async_event()
2036 ctrl->async_req.curr_bio = NULL; in nvme_tcp_submit_async_event()
2037 ctrl->async_req.data_len = 0; in nvme_tcp_submit_async_event()
2039 nvme_tcp_queue_request(&ctrl->async_req); in nvme_tcp_submit_async_event()
2046 struct nvme_tcp_ctrl *ctrl = req->queue->ctrl; in nvme_tcp_timeout() local
2054 if (ctrl->ctrl.state == NVME_CTRL_RESETTING) in nvme_tcp_timeout()
2057 dev_warn(ctrl->ctrl.device, in nvme_tcp_timeout()
2061 if (ctrl->ctrl.state != NVME_CTRL_LIVE) { in nvme_tcp_timeout()
2067 flush_work(&ctrl->err_work); in nvme_tcp_timeout()
2068 nvme_tcp_teardown_io_queues(&ctrl->ctrl, false); in nvme_tcp_timeout()
2069 nvme_tcp_teardown_admin_queue(&ctrl->ctrl, false); in nvme_tcp_timeout()
2073 dev_warn(ctrl->ctrl.device, "starting error recovery\n"); in nvme_tcp_timeout()
2074 nvme_tcp_error_recovery(&ctrl->ctrl); in nvme_tcp_timeout()
2140 dev_err(queue->ctrl->ctrl.device, in nvme_tcp_setup_cmd_pdu()
2158 if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) in nvme_tcp_queue_rq()
2159 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq); in nvme_tcp_queue_rq()
2174 struct nvme_tcp_ctrl *ctrl = set->driver_data; in nvme_tcp_map_queues() local
2175 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; in nvme_tcp_map_queues()
2177 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { in nvme_tcp_map_queues()
2180 ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_map_queues()
2183 ctrl->io_queues[HCTX_TYPE_READ]; in nvme_tcp_map_queues()
2185 ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_map_queues()
2189 ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_map_queues()
2192 ctrl->io_queues[HCTX_TYPE_DEFAULT]; in nvme_tcp_map_queues()
2198 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { in nvme_tcp_map_queues()
2201 ctrl->io_queues[HCTX_TYPE_POLL]; in nvme_tcp_map_queues()
2203 ctrl->io_queues[HCTX_TYPE_DEFAULT] + in nvme_tcp_map_queues()
2204 ctrl->io_queues[HCTX_TYPE_READ]; in nvme_tcp_map_queues()
2208 dev_info(ctrl->ctrl.device, in nvme_tcp_map_queues()
2210 ctrl->io_queues[HCTX_TYPE_DEFAULT], in nvme_tcp_map_queues()
2211 ctrl->io_queues[HCTX_TYPE_READ], in nvme_tcp_map_queues()
2212 ctrl->io_queues[HCTX_TYPE_POLL]); in nvme_tcp_map_queues()
2264 struct nvme_tcp_ctrl *ctrl; in nvme_tcp_existing_controller() local
2268 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { in nvme_tcp_existing_controller()
2269 found = nvmf_ip_options_match(&ctrl->ctrl, opts); in nvme_tcp_existing_controller()
2281 struct nvme_tcp_ctrl *ctrl; in nvme_tcp_create_ctrl() local
2284 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); in nvme_tcp_create_ctrl()
2285 if (!ctrl) in nvme_tcp_create_ctrl()
2288 INIT_LIST_HEAD(&ctrl->list); in nvme_tcp_create_ctrl()
2289 ctrl->ctrl.opts = opts; in nvme_tcp_create_ctrl()
2290 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + in nvme_tcp_create_ctrl()
2292 ctrl->ctrl.sqsize = opts->queue_size - 1; in nvme_tcp_create_ctrl()
2293 ctrl->ctrl.kato = opts->kato; in nvme_tcp_create_ctrl()
2295 INIT_DELAYED_WORK(&ctrl->connect_work, in nvme_tcp_create_ctrl()
2297 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); in nvme_tcp_create_ctrl()
2298 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); in nvme_tcp_create_ctrl()
2311 opts->traddr, opts->trsvcid, &ctrl->addr); in nvme_tcp_create_ctrl()
2320 opts->host_traddr, NULL, &ctrl->src_addr); in nvme_tcp_create_ctrl()
2333 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), in nvme_tcp_create_ctrl()
2335 if (!ctrl->queues) { in nvme_tcp_create_ctrl()
2340 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); in nvme_tcp_create_ctrl()
2344 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { in nvme_tcp_create_ctrl()
2350 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); in nvme_tcp_create_ctrl()
2354 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", in nvme_tcp_create_ctrl()
2355 ctrl->ctrl.opts->subsysnqn, &ctrl->addr); in nvme_tcp_create_ctrl()
2357 nvme_get_ctrl(&ctrl->ctrl); in nvme_tcp_create_ctrl()
2360 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); in nvme_tcp_create_ctrl()
2363 return &ctrl->ctrl; in nvme_tcp_create_ctrl()
2366 nvme_uninit_ctrl(&ctrl->ctrl); in nvme_tcp_create_ctrl()
2367 nvme_put_ctrl(&ctrl->ctrl); in nvme_tcp_create_ctrl()
2372 kfree(ctrl->queues); in nvme_tcp_create_ctrl()
2374 kfree(ctrl); in nvme_tcp_create_ctrl()
2403 struct nvme_tcp_ctrl *ctrl; in nvme_tcp_cleanup_module() local
2408 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) in nvme_tcp_cleanup_module()
2409 nvme_delete_ctrl(&ctrl->ctrl); in nvme_tcp_cleanup_module()