Lines Matching refs:udev
104 struct tcmu_dev *udev; member
298 struct tcmu_dev *udev = nl_cmd->udev; in tcmu_fail_netlink_cmd() local
306 pr_debug("Aborting nl cmd %d on %s\n", nl_cmd->cmd, udev->name); in tcmu_fail_netlink_cmd()
367 struct tcmu_dev *udev = NULL; in tcmu_genl_cmd_done() local
382 if (nl_cmd->udev->se_dev.dev_index == dev_id) { in tcmu_genl_cmd_done()
383 udev = nl_cmd->udev; in tcmu_genl_cmd_done()
388 if (!udev) { in tcmu_genl_cmd_done()
397 udev->name, dev_id, nl_cmd->cmd, completed_cmd, rc, in tcmu_genl_cmd_done()
402 udev->name, completed_cmd, nl_cmd->cmd); in tcmu_genl_cmd_done()
491 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev; in tcmu_cmd_free_data() local
495 clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap); in tcmu_cmd_free_data()
498 static inline int tcmu_get_empty_block(struct tcmu_dev *udev, in tcmu_get_empty_block() argument
505 dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh); in tcmu_get_empty_block()
506 if (dbi == udev->dbi_thresh) in tcmu_get_empty_block()
509 page = radix_tree_lookup(&udev->data_blocks, dbi); in tcmu_get_empty_block()
520 ret = radix_tree_insert(&udev->data_blocks, dbi, page); in tcmu_get_empty_block()
525 if (dbi > udev->dbi_max) in tcmu_get_empty_block()
526 udev->dbi_max = dbi; in tcmu_get_empty_block()
528 set_bit(dbi, udev->data_bitmap); in tcmu_get_empty_block()
542 static int tcmu_get_empty_blocks(struct tcmu_dev *udev, in tcmu_get_empty_blocks() argument
550 dbi = tcmu_get_empty_block(udev, tcmu_cmd, dbi, &iov_cnt); in tcmu_get_empty_blocks()
558 tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi) in tcmu_get_block_page() argument
560 return radix_tree_lookup(&udev->data_blocks, dbi); in tcmu_get_block_page()
588 static int new_block_to_iov(struct tcmu_dev *udev, struct tcmu_cmd *cmd, in new_block_to_iov() argument
607 (udev->data_off + dbi * DATA_BLOCK_SIZE); in new_block_to_iov()
614 static void tcmu_setup_iovs(struct tcmu_dev *udev, struct tcmu_cmd *cmd, in tcmu_setup_iovs() argument
622 dbi = new_block_to_iov(udev, cmd, iov, dbi, &data_length); in tcmu_setup_iovs()
628 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_alloc_cmd() local
637 tcmu_cmd->tcmu_dev = udev; in tcmu_alloc_cmd()
691 static void scatter_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd, in scatter_data_area() argument
715 dbi = new_block_to_iov(udev, tcmu_cmd, iov, dbi, in scatter_data_area()
717 page = tcmu_get_block_page(udev, dbi); in scatter_data_area()
740 static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd, in gather_data_area() argument
780 page = tcmu_get_block_page(udev, dbi); in gather_data_area()
814 static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size) in is_ring_space_avail() argument
816 struct tcmu_mailbox *mb = udev->mb_addr; in is_ring_space_avail()
822 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ in is_ring_space_avail()
828 if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size) in is_ring_space_avail()
831 cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size); in is_ring_space_avail()
833 space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size); in is_ring_space_avail()
836 udev->cmdr_last_cleaned, udev->cmdr_size); in is_ring_space_avail()
848 static int tcmu_alloc_data_space(struct tcmu_dev *udev, struct tcmu_cmd *cmd, in tcmu_alloc_data_space() argument
857 space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh); in tcmu_alloc_data_space()
860 (udev->max_blocks - udev->dbi_thresh) + space; in tcmu_alloc_data_space()
869 udev->dbi_thresh += cmd->dbi_cnt; in tcmu_alloc_data_space()
870 if (udev->dbi_thresh > udev->max_blocks) in tcmu_alloc_data_space()
871 udev->dbi_thresh = udev->max_blocks; in tcmu_alloc_data_space()
874 iov_cnt = tcmu_get_empty_blocks(udev, cmd, in tcmu_alloc_data_space()
880 ret = tcmu_get_empty_blocks(udev, cmd, cmd->dbi_bidi_cnt); in tcmu_alloc_data_space()
926 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev; in add_to_qfull_queue() local
933 if (!udev->qfull_time_out) in add_to_qfull_queue()
935 else if (udev->qfull_time_out > 0) in add_to_qfull_queue()
936 tmo = udev->qfull_time_out; in add_to_qfull_queue()
937 else if (udev->cmd_time_out) in add_to_qfull_queue()
938 tmo = udev->cmd_time_out; in add_to_qfull_queue()
942 tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer); in add_to_qfull_queue()
944 list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue); in add_to_qfull_queue()
946 tcmu_cmd, udev->name); in add_to_qfull_queue()
950 static uint32_t ring_insert_padding(struct tcmu_dev *udev, size_t cmd_size) in ring_insert_padding() argument
953 struct tcmu_mailbox *mb = udev->mb_addr; in ring_insert_padding()
954 uint32_t cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ in ring_insert_padding()
957 if (head_to_end(cmd_head, udev->cmdr_size) < cmd_size) { in ring_insert_padding()
958 size_t pad_size = head_to_end(cmd_head, udev->cmdr_size); in ring_insert_padding()
968 UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size); in ring_insert_padding()
971 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ in ring_insert_padding()
990 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev; in queue_cmd_ring() local
993 struct tcmu_mailbox *mb = udev->mb_addr; in queue_cmd_ring()
1004 if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) { in queue_cmd_ring()
1009 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) { in queue_cmd_ring()
1014 if (!list_empty(&udev->qfull_queue)) in queue_cmd_ring()
1017 if (data_length > udev->data_size) { in queue_cmd_ring()
1019 data_length, udev->data_size); in queue_cmd_ring()
1024 iov_cnt = tcmu_alloc_data_space(udev, tcmu_cmd, &iov_bidi_cnt); in queue_cmd_ring()
1035 if (command_size > (udev->cmdr_size / 2)) { in queue_cmd_ring()
1037 command_size, udev->cmdr_size); in queue_cmd_ring()
1043 if (!is_ring_space_avail(udev, command_size)) in queue_cmd_ring()
1050 cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 1, USHRT_MAX, GFP_NOWAIT); in queue_cmd_ring()
1061 tcmu_cmd, udev->name); in queue_cmd_ring()
1063 cmd_head = ring_insert_padding(udev, command_size); in queue_cmd_ring()
1075 scatter_data_area(udev, tcmu_cmd, &iov); in queue_cmd_ring()
1077 tcmu_setup_iovs(udev, tcmu_cmd, &iov, se_cmd->data_length); in queue_cmd_ring()
1084 tcmu_setup_iovs(udev, tcmu_cmd, &iov, tcmu_cmd->data_len_bidi); in queue_cmd_ring()
1088 tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer); in queue_cmd_ring()
1100 UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size); in queue_cmd_ring()
1103 list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue); in queue_cmd_ring()
1106 uio_event_notify(&udev->uio_info); in queue_cmd_ring()
1133 queue_tmr_ring(struct tcmu_dev *udev, struct tcmu_tmr *tmr) in queue_tmr_ring() argument
1138 struct tcmu_mailbox *mb = udev->mb_addr; in queue_tmr_ring()
1141 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) in queue_tmr_ring()
1147 if (!list_empty(&udev->tmr_queue) || in queue_tmr_ring()
1148 !is_ring_space_avail(udev, cmd_size)) { in queue_tmr_ring()
1149 list_add_tail(&tmr->queue_entry, &udev->tmr_queue); in queue_tmr_ring()
1151 tmr, udev->name); in queue_tmr_ring()
1155 cmd_head = ring_insert_padding(udev, cmd_size); in queue_tmr_ring()
1166 UPDATE_HEAD(mb->cmd_head, cmd_size, udev->cmdr_size); in queue_tmr_ring()
1169 uio_event_notify(&udev->uio_info); in queue_tmr_ring()
1181 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_queue_cmd() local
1190 mutex_lock(&udev->cmdr_lock); in tcmu_queue_cmd()
1196 mutex_unlock(&udev->cmdr_lock); in tcmu_queue_cmd()
1238 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_tmr_notify() local
1240 mutex_lock(&udev->cmdr_lock); in tcmu_tmr_notify()
1254 cmd, udev->name); in tcmu_tmr_notify()
1262 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer); in tcmu_tmr_notify()
1264 if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags)) in tcmu_tmr_notify()
1268 tcmu_tmr_type(tmf), udev->name, i, cmd_cnt); in tcmu_tmr_notify()
1288 queue_tmr_ring(udev, tmr); in tcmu_tmr_notify()
1291 mutex_unlock(&udev->cmdr_lock); in tcmu_tmr_notify()
1297 struct tcmu_dev *udev = cmd->tcmu_dev; in tcmu_handle_completion() local
1338 gather_data_area(udev, cmd, true, read_len); in tcmu_handle_completion()
1340 gather_data_area(udev, cmd, false, read_len); in tcmu_handle_completion()
1361 static int tcmu_run_tmr_queue(struct tcmu_dev *udev) in tcmu_run_tmr_queue() argument
1366 if (list_empty(&udev->tmr_queue)) in tcmu_run_tmr_queue()
1369 pr_debug("running %s's tmr queue\n", udev->name); in tcmu_run_tmr_queue()
1371 list_splice_init(&udev->tmr_queue, &tmrs); in tcmu_run_tmr_queue()
1377 tmr, udev->name); in tcmu_run_tmr_queue()
1379 if (queue_tmr_ring(udev, tmr)) { in tcmu_run_tmr_queue()
1385 list_splice_tail(&tmrs, &udev->tmr_queue); in tcmu_run_tmr_queue()
1393 static unsigned int tcmu_handle_completions(struct tcmu_dev *udev) in tcmu_handle_completions() argument
1399 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) { in tcmu_handle_completions()
1404 mb = udev->mb_addr; in tcmu_handle_completions()
1407 while (udev->cmdr_last_cleaned != READ_ONCE(mb->cmd_tail)) { in tcmu_handle_completions()
1409 struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned; in tcmu_handle_completions()
1415 size_t ring_left = head_to_end(udev->cmdr_last_cleaned, in tcmu_handle_completions()
1416 udev->cmdr_size); in tcmu_handle_completions()
1424 UPDATE_HEAD(udev->cmdr_last_cleaned, in tcmu_handle_completions()
1426 udev->cmdr_size); in tcmu_handle_completions()
1431 cmd = idr_remove(&udev->commands, entry->hdr.cmd_id); in tcmu_handle_completions()
1435 set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags); in tcmu_handle_completions()
1441 UPDATE_HEAD(udev->cmdr_last_cleaned, in tcmu_handle_completions()
1443 udev->cmdr_size); in tcmu_handle_completions()
1446 free_space = tcmu_run_tmr_queue(udev); in tcmu_handle_completions()
1449 idr_is_empty(&udev->commands) && list_empty(&udev->qfull_queue)) { in tcmu_handle_completions()
1456 if (udev->cmd_time_out) in tcmu_handle_completions()
1457 tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer); in tcmu_handle_completions()
1498 static void tcmu_device_timedout(struct tcmu_dev *udev) in tcmu_device_timedout() argument
1501 if (list_empty(&udev->timedout_entry)) in tcmu_device_timedout()
1502 list_add_tail(&udev->timedout_entry, &timed_out_udevs); in tcmu_device_timedout()
1510 struct tcmu_dev *udev = from_timer(udev, t, cmd_timer); in tcmu_cmd_timedout() local
1512 pr_debug("%s cmd timeout has expired\n", udev->name); in tcmu_cmd_timedout()
1513 tcmu_device_timedout(udev); in tcmu_cmd_timedout()
1518 struct tcmu_dev *udev = from_timer(udev, t, qfull_timer); in tcmu_qfull_timedout() local
1520 pr_debug("%s qfull timeout has expired\n", udev->name); in tcmu_qfull_timedout()
1521 tcmu_device_timedout(udev); in tcmu_qfull_timedout()
1546 struct tcmu_dev *udev; in tcmu_alloc_device() local
1548 udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL); in tcmu_alloc_device()
1549 if (!udev) in tcmu_alloc_device()
1551 kref_init(&udev->kref); in tcmu_alloc_device()
1553 udev->name = kstrdup(name, GFP_KERNEL); in tcmu_alloc_device()
1554 if (!udev->name) { in tcmu_alloc_device()
1555 kfree(udev); in tcmu_alloc_device()
1559 udev->hba = hba; in tcmu_alloc_device()
1560 udev->cmd_time_out = TCMU_TIME_OUT; in tcmu_alloc_device()
1561 udev->qfull_time_out = -1; in tcmu_alloc_device()
1563 udev->max_blocks = DATA_BLOCK_BITS_DEF; in tcmu_alloc_device()
1564 mutex_init(&udev->cmdr_lock); in tcmu_alloc_device()
1566 INIT_LIST_HEAD(&udev->node); in tcmu_alloc_device()
1567 INIT_LIST_HEAD(&udev->timedout_entry); in tcmu_alloc_device()
1568 INIT_LIST_HEAD(&udev->qfull_queue); in tcmu_alloc_device()
1569 INIT_LIST_HEAD(&udev->tmr_queue); in tcmu_alloc_device()
1570 INIT_LIST_HEAD(&udev->inflight_queue); in tcmu_alloc_device()
1571 idr_init(&udev->commands); in tcmu_alloc_device()
1573 timer_setup(&udev->qfull_timer, tcmu_qfull_timedout, 0); in tcmu_alloc_device()
1574 timer_setup(&udev->cmd_timer, tcmu_cmd_timedout, 0); in tcmu_alloc_device()
1576 INIT_RADIX_TREE(&udev->data_blocks, GFP_KERNEL); in tcmu_alloc_device()
1578 return &udev->se_dev; in tcmu_alloc_device()
1581 static void run_qfull_queue(struct tcmu_dev *udev, bool fail) in run_qfull_queue() argument
1588 if (list_empty(&udev->qfull_queue)) in run_qfull_queue()
1591 pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail); in run_qfull_queue()
1593 list_splice_init(&udev->qfull_queue, &cmds); in run_qfull_queue()
1599 tcmu_cmd, udev->name); in run_qfull_queue()
1617 tcmu_cmd, udev->name, scsi_ret); in run_qfull_queue()
1631 list_splice_tail(&cmds, &udev->qfull_queue); in run_qfull_queue()
1636 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer); in run_qfull_queue()
1641 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); in tcmu_irqcontrol() local
1643 mutex_lock(&udev->cmdr_lock); in tcmu_irqcontrol()
1644 if (tcmu_handle_completions(udev)) in tcmu_irqcontrol()
1645 run_qfull_queue(udev, false); in tcmu_irqcontrol()
1646 mutex_unlock(&udev->cmdr_lock); in tcmu_irqcontrol()
1657 struct tcmu_dev *udev = vma->vm_private_data; in tcmu_find_mem_index() local
1658 struct uio_info *info = &udev->uio_info; in tcmu_find_mem_index()
1668 static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi) in tcmu_try_get_block_page() argument
1672 mutex_lock(&udev->cmdr_lock); in tcmu_try_get_block_page()
1673 page = tcmu_get_block_page(udev, dbi); in tcmu_try_get_block_page()
1675 mutex_unlock(&udev->cmdr_lock); in tcmu_try_get_block_page()
1684 dbi, udev->name); in tcmu_try_get_block_page()
1686 mutex_unlock(&udev->cmdr_lock); in tcmu_try_get_block_page()
1693 struct tcmu_dev *udev = vmf->vma->vm_private_data; in tcmu_vma_fault() local
1694 struct uio_info *info = &udev->uio_info; in tcmu_vma_fault()
1709 if (offset < udev->data_off) { in tcmu_vma_fault()
1717 dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE; in tcmu_vma_fault()
1718 page = tcmu_try_get_block_page(udev, dbi); in tcmu_vma_fault()
1734 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); in tcmu_mmap() local
1739 vma->vm_private_data = udev; in tcmu_mmap()
1742 if (vma_pages(vma) != (udev->ring_size >> PAGE_SHIFT)) in tcmu_mmap()
1750 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); in tcmu_open() local
1753 if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags)) in tcmu_open()
1756 udev->inode = inode; in tcmu_open()
1757 kref_get(&udev->kref); in tcmu_open()
1767 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_dev_call_rcu() local
1769 kfree(udev->uio_info.name); in tcmu_dev_call_rcu()
1770 kfree(udev->name); in tcmu_dev_call_rcu()
1771 kfree(udev); in tcmu_dev_call_rcu()
1798 static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev) in tcmu_remove_all_queued_tmr() argument
1802 list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) { in tcmu_remove_all_queued_tmr()
1810 struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref); in tcmu_dev_kref_release() local
1811 struct se_device *dev = &udev->se_dev; in tcmu_dev_kref_release()
1816 vfree(udev->mb_addr); in tcmu_dev_kref_release()
1817 udev->mb_addr = NULL; in tcmu_dev_kref_release()
1820 if (!list_empty(&udev->timedout_entry)) in tcmu_dev_kref_release()
1821 list_del(&udev->timedout_entry); in tcmu_dev_kref_release()
1825 mutex_lock(&udev->cmdr_lock); in tcmu_dev_kref_release()
1826 idr_for_each_entry(&udev->commands, cmd, i) { in tcmu_dev_kref_release()
1831 tcmu_remove_all_queued_tmr(udev); in tcmu_dev_kref_release()
1832 if (!list_empty(&udev->qfull_queue)) in tcmu_dev_kref_release()
1834 idr_destroy(&udev->commands); in tcmu_dev_kref_release()
1837 tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1); in tcmu_dev_kref_release()
1838 bitmap_free(udev->data_bitmap); in tcmu_dev_kref_release()
1839 mutex_unlock(&udev->cmdr_lock); in tcmu_dev_kref_release()
1846 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); in tcmu_release() local
1848 clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags); in tcmu_release()
1852 kref_put(&udev->kref, tcmu_dev_kref_release); in tcmu_release()
1856 static int tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd) in tcmu_init_genl_cmd_reply() argument
1858 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd; in tcmu_init_genl_cmd_reply()
1863 if (udev->nl_reply_supported <= 0) in tcmu_init_genl_cmd_reply()
1871 udev->name); in tcmu_init_genl_cmd_reply()
1878 nl_cmd->cmd, udev->name); in tcmu_init_genl_cmd_reply()
1884 nl_cmd->udev = udev; in tcmu_init_genl_cmd_reply()
1894 static void tcmu_destroy_genl_cmd_reply(struct tcmu_dev *udev) in tcmu_destroy_genl_cmd_reply() argument
1896 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd; in tcmu_destroy_genl_cmd_reply()
1901 if (udev->nl_reply_supported <= 0) in tcmu_destroy_genl_cmd_reply()
1912 static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev) in tcmu_wait_genl_cmd_reply() argument
1914 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd; in tcmu_wait_genl_cmd_reply()
1920 if (udev->nl_reply_supported <= 0) in tcmu_wait_genl_cmd_reply()
1934 static int tcmu_netlink_event_init(struct tcmu_dev *udev, in tcmu_netlink_event_init() argument
1950 ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name); in tcmu_netlink_event_init()
1954 ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor); in tcmu_netlink_event_init()
1958 ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index); in tcmu_netlink_event_init()
1971 static int tcmu_netlink_event_send(struct tcmu_dev *udev, in tcmu_netlink_event_send() argument
1979 ret = tcmu_init_genl_cmd_reply(udev, cmd); in tcmu_netlink_event_send()
1991 return tcmu_wait_genl_cmd_reply(udev); in tcmu_netlink_event_send()
1993 tcmu_destroy_genl_cmd_reply(udev); in tcmu_netlink_event_send()
1998 static int tcmu_send_dev_add_event(struct tcmu_dev *udev) in tcmu_send_dev_add_event() argument
2004 ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb, in tcmu_send_dev_add_event()
2008 return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, skb, in tcmu_send_dev_add_event()
2012 static int tcmu_send_dev_remove_event(struct tcmu_dev *udev) in tcmu_send_dev_remove_event() argument
2018 ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE, in tcmu_send_dev_remove_event()
2022 return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE, in tcmu_send_dev_remove_event()
2026 static int tcmu_update_uio_info(struct tcmu_dev *udev) in tcmu_update_uio_info() argument
2028 struct tcmu_hba *hba = udev->hba->hba_ptr; in tcmu_update_uio_info()
2032 info = &udev->uio_info; in tcmu_update_uio_info()
2034 if (udev->dev_config[0]) in tcmu_update_uio_info()
2036 udev->name, udev->dev_config); in tcmu_update_uio_info()
2039 udev->name); in tcmu_update_uio_info()
2052 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_configure_device() local
2057 ret = tcmu_update_uio_info(udev); in tcmu_configure_device()
2061 info = &udev->uio_info; in tcmu_configure_device()
2063 mutex_lock(&udev->cmdr_lock); in tcmu_configure_device()
2064 udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL); in tcmu_configure_device()
2065 mutex_unlock(&udev->cmdr_lock); in tcmu_configure_device()
2066 if (!udev->data_bitmap) { in tcmu_configure_device()
2071 udev->mb_addr = vzalloc(CMDR_SIZE); in tcmu_configure_device()
2072 if (!udev->mb_addr) { in tcmu_configure_device()
2078 udev->cmdr_size = CMDR_SIZE - CMDR_OFF; in tcmu_configure_device()
2079 udev->data_off = CMDR_SIZE; in tcmu_configure_device()
2080 udev->data_size = udev->max_blocks * DATA_BLOCK_SIZE; in tcmu_configure_device()
2081 udev->dbi_thresh = 0; /* Default in Idle state */ in tcmu_configure_device()
2084 mb = udev->mb_addr; in tcmu_configure_device()
2090 mb->cmdr_size = udev->cmdr_size; in tcmu_configure_device()
2092 WARN_ON(!PAGE_ALIGNED(udev->data_off)); in tcmu_configure_device()
2093 WARN_ON(udev->data_size % PAGE_SIZE); in tcmu_configure_device()
2094 WARN_ON(udev->data_size % DATA_BLOCK_SIZE); in tcmu_configure_device()
2099 info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr; in tcmu_configure_device()
2100 info->mem[0].size = udev->ring_size = udev->data_size + CMDR_SIZE; in tcmu_configure_device()
2127 if (udev->nl_reply_supported >= 0) in tcmu_configure_device()
2128 udev->nl_reply_supported = tcmu_kern_cmd_reply_supported; in tcmu_configure_device()
2134 kref_get(&udev->kref); in tcmu_configure_device()
2136 ret = tcmu_send_dev_add_event(udev); in tcmu_configure_device()
2141 list_add(&udev->node, &root_udev); in tcmu_configure_device()
2147 kref_put(&udev->kref, tcmu_dev_kref_release); in tcmu_configure_device()
2148 uio_unregister_device(&udev->uio_info); in tcmu_configure_device()
2150 vfree(udev->mb_addr); in tcmu_configure_device()
2151 udev->mb_addr = NULL; in tcmu_configure_device()
2153 bitmap_free(udev->data_bitmap); in tcmu_configure_device()
2154 udev->data_bitmap = NULL; in tcmu_configure_device()
2164 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_free_device() local
2167 kref_put(&udev->kref, tcmu_dev_kref_release); in tcmu_free_device()
2172 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_destroy_device() local
2174 del_timer_sync(&udev->cmd_timer); in tcmu_destroy_device()
2175 del_timer_sync(&udev->qfull_timer); in tcmu_destroy_device()
2178 list_del(&udev->node); in tcmu_destroy_device()
2181 tcmu_send_dev_remove_event(udev); in tcmu_destroy_device()
2183 uio_unregister_device(&udev->uio_info); in tcmu_destroy_device()
2186 kref_put(&udev->kref, tcmu_dev_kref_release); in tcmu_destroy_device()
2189 static void tcmu_unblock_dev(struct tcmu_dev *udev) in tcmu_unblock_dev() argument
2191 mutex_lock(&udev->cmdr_lock); in tcmu_unblock_dev()
2192 clear_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags); in tcmu_unblock_dev()
2193 mutex_unlock(&udev->cmdr_lock); in tcmu_unblock_dev()
2196 static void tcmu_block_dev(struct tcmu_dev *udev) in tcmu_block_dev() argument
2198 mutex_lock(&udev->cmdr_lock); in tcmu_block_dev()
2200 if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) in tcmu_block_dev()
2204 tcmu_handle_completions(udev); in tcmu_block_dev()
2206 run_qfull_queue(udev, true); in tcmu_block_dev()
2209 mutex_unlock(&udev->cmdr_lock); in tcmu_block_dev()
2212 static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level) in tcmu_reset_ring() argument
2218 mutex_lock(&udev->cmdr_lock); in tcmu_reset_ring()
2220 idr_for_each_entry(&udev->commands, cmd, i) { in tcmu_reset_ring()
2222 cmd->cmd_id, udev->name, in tcmu_reset_ring()
2225 idr_remove(&udev->commands, i); in tcmu_reset_ring()
2245 mb = udev->mb_addr; in tcmu_reset_ring()
2247 pr_debug("mb last %u head %u tail %u\n", udev->cmdr_last_cleaned, in tcmu_reset_ring()
2250 udev->cmdr_last_cleaned = 0; in tcmu_reset_ring()
2254 clear_bit(TCMU_DEV_BIT_BROKEN, &udev->flags); in tcmu_reset_ring()
2256 del_timer(&udev->cmd_timer); in tcmu_reset_ring()
2265 tcmu_remove_all_queued_tmr(udev); in tcmu_reset_ring()
2267 run_qfull_queue(udev, false); in tcmu_reset_ring()
2269 mutex_unlock(&udev->cmdr_lock); in tcmu_reset_ring()
2307 static int tcmu_set_max_blocks_param(struct tcmu_dev *udev, substring_t *arg) in tcmu_set_max_blocks_param() argument
2323 mutex_lock(&udev->cmdr_lock); in tcmu_set_max_blocks_param()
2324 if (udev->data_bitmap) { in tcmu_set_max_blocks_param()
2330 udev->max_blocks = TCMU_MBS_TO_BLOCKS(val); in tcmu_set_max_blocks_param()
2331 if (udev->max_blocks > tcmu_global_max_blocks) { in tcmu_set_max_blocks_param()
2334 udev->max_blocks = tcmu_global_max_blocks; in tcmu_set_max_blocks_param()
2338 mutex_unlock(&udev->cmdr_lock); in tcmu_set_max_blocks_param()
2345 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_set_configfs_dev_params() local
2363 if (match_strlcpy(udev->dev_config, &args[0], in tcmu_set_configfs_dev_params()
2368 pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config); in tcmu_set_configfs_dev_params()
2371 ret = match_u64(&args[0], &udev->dev_size); in tcmu_set_configfs_dev_params()
2385 ret = match_int(&args[0], &udev->nl_reply_supported); in tcmu_set_configfs_dev_params()
2391 ret = tcmu_set_max_blocks_param(udev, &args[0]); in tcmu_set_configfs_dev_params()
2407 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_show_configfs_dev_params() local
2411 udev->dev_config[0] ? udev->dev_config : "NULL"); in tcmu_show_configfs_dev_params()
2412 bl += sprintf(b + bl, "Size: %llu ", udev->dev_size); in tcmu_show_configfs_dev_params()
2414 TCMU_BLOCKS_TO_MBS(udev->max_blocks)); in tcmu_show_configfs_dev_params()
2421 struct tcmu_dev *udev = TCMU_DEV(dev); in tcmu_get_blocks() local
2423 return div_u64(udev->dev_size - dev->dev_attrib.block_size, in tcmu_get_blocks()
2437 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_cmd_time_out_show() local
2439 return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC); in tcmu_cmd_time_out_show()
2447 struct tcmu_dev *udev = container_of(da->da_dev, in tcmu_cmd_time_out_store() local
2461 udev->cmd_time_out = val * MSEC_PER_SEC; in tcmu_cmd_time_out_store()
2470 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_qfull_time_out_show() local
2472 return snprintf(page, PAGE_SIZE, "%ld\n", udev->qfull_time_out <= 0 ? in tcmu_qfull_time_out_show()
2473 udev->qfull_time_out : in tcmu_qfull_time_out_show()
2474 udev->qfull_time_out / MSEC_PER_SEC); in tcmu_qfull_time_out_show()
2482 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_qfull_time_out_store() local
2491 udev->qfull_time_out = val * MSEC_PER_SEC; in tcmu_qfull_time_out_store()
2493 udev->qfull_time_out = val; in tcmu_qfull_time_out_store()
2506 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_max_data_area_mb_show() local
2509 TCMU_BLOCKS_TO_MBS(udev->max_blocks)); in tcmu_max_data_area_mb_show()
2517 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_dev_config_show() local
2519 return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config); in tcmu_dev_config_show()
2522 static int tcmu_send_dev_config_event(struct tcmu_dev *udev, in tcmu_send_dev_config_event() argument
2529 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_dev_config_event()
2538 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_dev_config_event()
2548 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_dev_config_store() local
2556 if (target_dev_configured(&udev->se_dev)) { in tcmu_dev_config_store()
2557 ret = tcmu_send_dev_config_event(udev, page); in tcmu_dev_config_store()
2562 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN); in tcmu_dev_config_store()
2564 ret = tcmu_update_uio_info(udev); in tcmu_dev_config_store()
2569 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN); in tcmu_dev_config_store()
2579 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_dev_size_show() local
2581 return snprintf(page, PAGE_SIZE, "%llu\n", udev->dev_size); in tcmu_dev_size_show()
2584 static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size) in tcmu_send_dev_size_event() argument
2590 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_dev_size_event()
2600 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_dev_size_event()
2609 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_dev_size_store() local
2618 if (target_dev_configured(&udev->se_dev)) { in tcmu_dev_size_store()
2619 ret = tcmu_send_dev_size_event(udev, val); in tcmu_dev_size_store()
2625 udev->dev_size = val; in tcmu_dev_size_store()
2635 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_nl_reply_supported_show() local
2637 return snprintf(page, PAGE_SIZE, "%d\n", udev->nl_reply_supported); in tcmu_nl_reply_supported_show()
2645 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_nl_reply_supported_store() local
2653 udev->nl_reply_supported = val; in tcmu_nl_reply_supported_store()
2667 static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val) in tcmu_send_emulate_write_cache() argument
2673 ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_emulate_write_cache()
2682 return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE, in tcmu_send_emulate_write_cache()
2691 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_emulate_write_cache_store() local
2700 if (target_dev_configured(&udev->se_dev)) { in tcmu_emulate_write_cache_store()
2701 ret = tcmu_send_emulate_write_cache(udev, val); in tcmu_emulate_write_cache_store()
2717 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_tmr_notification_show() local
2720 test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags)); in tcmu_tmr_notification_show()
2728 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); in tcmu_tmr_notification_store() local
2739 set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags); in tcmu_tmr_notification_store()
2741 clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags); in tcmu_tmr_notification_store()
2751 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_block_dev_show() local
2753 if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) in tcmu_block_dev_show()
2765 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_block_dev_store() local
2769 if (!target_dev_configured(&udev->se_dev)) { in tcmu_block_dev_store()
2784 tcmu_unblock_dev(udev); in tcmu_block_dev_store()
2786 tcmu_block_dev(udev); in tcmu_block_dev_store()
2797 struct tcmu_dev *udev = TCMU_DEV(se_dev); in tcmu_reset_ring_store() local
2801 if (!target_dev_configured(&udev->se_dev)) { in tcmu_reset_ring_store()
2815 tcmu_reset_ring(udev, val); in tcmu_reset_ring_store()
2863 struct tcmu_dev *udev; in find_free_blocks() local
2871 list_for_each_entry(udev, &root_udev, node) { in find_free_blocks()
2872 mutex_lock(&udev->cmdr_lock); in find_free_blocks()
2874 if (!target_dev_configured(&udev->se_dev)) { in find_free_blocks()
2875 mutex_unlock(&udev->cmdr_lock); in find_free_blocks()
2880 if (tcmu_handle_completions(udev)) in find_free_blocks()
2881 run_qfull_queue(udev, false); in find_free_blocks()
2884 if (!udev->dbi_thresh) { in find_free_blocks()
2885 mutex_unlock(&udev->cmdr_lock); in find_free_blocks()
2889 end = udev->dbi_max + 1; in find_free_blocks()
2890 block = find_last_bit(udev->data_bitmap, end); in find_free_blocks()
2891 if (block == udev->dbi_max) { in find_free_blocks()
2896 mutex_unlock(&udev->cmdr_lock); in find_free_blocks()
2900 udev->dbi_thresh = start = 0; in find_free_blocks()
2901 udev->dbi_max = 0; in find_free_blocks()
2903 udev->dbi_thresh = start = block + 1; in find_free_blocks()
2904 udev->dbi_max = block; in find_free_blocks()
2908 off = udev->data_off + start * DATA_BLOCK_SIZE; in find_free_blocks()
2909 unmap_mapping_range(udev->inode->i_mapping, off, 0, 1); in find_free_blocks()
2912 tcmu_blocks_release(&udev->data_blocks, start, end); in find_free_blocks()
2913 mutex_unlock(&udev->cmdr_lock); in find_free_blocks()
2917 total_freed, udev->name); in find_free_blocks()
2927 struct tcmu_dev *udev, *tmp_dev; in check_timedout_devices() local
2934 list_for_each_entry_safe(udev, tmp_dev, &devs, timedout_entry) { in check_timedout_devices()
2935 list_del_init(&udev->timedout_entry); in check_timedout_devices()
2938 mutex_lock(&udev->cmdr_lock); in check_timedout_devices()
2944 if (udev->cmd_time_out) { in check_timedout_devices()
2946 &udev->inflight_queue, in check_timedout_devices()
2950 tcmu_set_next_deadline(&udev->inflight_queue, in check_timedout_devices()
2951 &udev->cmd_timer); in check_timedout_devices()
2953 list_for_each_entry_safe(cmd, tmp_cmd, &udev->qfull_queue, in check_timedout_devices()
2957 tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer); in check_timedout_devices()
2959 mutex_unlock(&udev->cmdr_lock); in check_timedout_devices()