Lines Matching refs:ub
197 static inline bool ublk_dev_is_user_copy(const struct ublk_device *ub) in ublk_dev_is_user_copy() argument
199 return ub->dev_info.flags & UBLK_F_USER_COPY; in ublk_dev_is_user_copy()
202 static inline bool ublk_dev_is_zoned(const struct ublk_device *ub) in ublk_dev_is_zoned() argument
204 return ub->dev_info.flags & UBLK_F_ZONED; in ublk_dev_is_zoned()
214 static int ublk_get_nr_zones(const struct ublk_device *ub) in ublk_get_nr_zones() argument
216 const struct ublk_param_basic *p = &ub->params.basic; in ublk_get_nr_zones()
222 static int ublk_revalidate_disk_zones(struct ublk_device *ub) in ublk_revalidate_disk_zones() argument
224 return blk_revalidate_disk_zones(ub->ub_disk, NULL); in ublk_revalidate_disk_zones()
227 static int ublk_dev_param_zoned_validate(const struct ublk_device *ub) in ublk_dev_param_zoned_validate() argument
229 const struct ublk_param_zoned *p = &ub->params.zoned; in ublk_dev_param_zoned_validate()
232 if (!ublk_dev_is_zoned(ub)) in ublk_dev_param_zoned_validate()
238 nr_zones = ublk_get_nr_zones(ub); in ublk_dev_param_zoned_validate()
249 static int ublk_dev_param_zoned_apply(struct ublk_device *ub) in ublk_dev_param_zoned_apply() argument
251 const struct ublk_param_zoned *p = &ub->params.zoned; in ublk_dev_param_zoned_apply()
253 disk_set_zoned(ub->ub_disk, BLK_ZONED_HM); in ublk_dev_param_zoned_apply()
254 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, ub->ub_disk->queue); in ublk_dev_param_zoned_apply()
255 blk_queue_required_elevator_features(ub->ub_disk->queue, in ublk_dev_param_zoned_apply()
257 disk_set_max_active_zones(ub->ub_disk, p->max_active_zones); in ublk_dev_param_zoned_apply()
258 disk_set_max_open_zones(ub->ub_disk, p->max_open_zones); in ublk_dev_param_zoned_apply()
259 blk_queue_max_zone_append_sectors(ub->ub_disk->queue, p->max_zone_append_sectors); in ublk_dev_param_zoned_apply()
261 ub->ub_disk->nr_zones = ublk_get_nr_zones(ub); in ublk_dev_param_zoned_apply()
297 struct ublk_device *ub = disk->private_data; in ublk_report_zones() local
306 nr_zones = min_t(unsigned int, ub->ub_disk->nr_zones - first_zone, in ublk_report_zones()
309 buffer = ublk_alloc_report_buffer(ub, nr_zones, &buffer_length); in ublk_report_zones()
430 static int ublk_dev_param_zoned_validate(const struct ublk_device *ub) in ublk_dev_param_zoned_validate() argument
435 static int ublk_dev_param_zoned_apply(struct ublk_device *ub) in ublk_dev_param_zoned_apply() argument
440 static int ublk_revalidate_disk_zones(struct ublk_device *ub) in ublk_revalidate_disk_zones() argument
495 static void ublk_dev_param_basic_apply(struct ublk_device *ub) in ublk_dev_param_basic_apply() argument
497 struct request_queue *q = ub->ub_disk->queue; in ublk_dev_param_basic_apply()
498 const struct ublk_param_basic *p = &ub->params.basic; in ublk_dev_param_basic_apply()
517 set_disk_ro(ub->ub_disk, true); in ublk_dev_param_basic_apply()
519 set_capacity(ub->ub_disk, p->dev_sectors); in ublk_dev_param_basic_apply()
522 static void ublk_dev_param_discard_apply(struct ublk_device *ub) in ublk_dev_param_discard_apply() argument
524 struct request_queue *q = ub->ub_disk->queue; in ublk_dev_param_discard_apply()
525 const struct ublk_param_discard *p = &ub->params.discard; in ublk_dev_param_discard_apply()
535 static int ublk_validate_params(const struct ublk_device *ub) in ublk_validate_params() argument
538 if (ub->params.types & UBLK_PARAM_TYPE_BASIC) { in ublk_validate_params()
539 const struct ublk_param_basic *p = &ub->params.basic; in ublk_validate_params()
547 if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9)) in ublk_validate_params()
550 if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) in ublk_validate_params()
555 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) { in ublk_validate_params()
556 const struct ublk_param_discard *p = &ub->params.discard; in ublk_validate_params()
567 if (ub->params.types & UBLK_PARAM_TYPE_DEVT) in ublk_validate_params()
570 if (ub->params.types & UBLK_PARAM_TYPE_ZONED) in ublk_validate_params()
571 return ublk_dev_param_zoned_validate(ub); in ublk_validate_params()
572 else if (ublk_dev_is_zoned(ub)) in ublk_validate_params()
578 static int ublk_apply_params(struct ublk_device *ub) in ublk_apply_params() argument
580 if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC)) in ublk_apply_params()
583 ublk_dev_param_basic_apply(ub); in ublk_apply_params()
585 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) in ublk_apply_params()
586 ublk_dev_param_discard_apply(ub); in ublk_apply_params()
588 if (ub->params.types & UBLK_PARAM_TYPE_ZONED) in ublk_apply_params()
589 return ublk_dev_param_zoned_apply(ub); in ublk_apply_params()
647 static struct ublk_device *ublk_get_device(struct ublk_device *ub) in ublk_get_device() argument
649 if (kobject_get_unless_zero(&ub->cdev_dev.kobj)) in ublk_get_device()
650 return ub; in ublk_get_device()
654 static void ublk_put_device(struct ublk_device *ub) in ublk_put_device() argument
656 put_device(&ub->cdev_dev); in ublk_put_device()
677 static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id) in ublk_queue_cmd_buf() argument
679 return ublk_get_queue(ub, q_id)->io_cmd_buf; in ublk_queue_cmd_buf()
682 static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id) in ublk_queue_cmd_buf_size() argument
684 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_queue_cmd_buf_size()
703 static inline bool ublk_can_use_recovery(struct ublk_device *ub) in ublk_can_use_recovery() argument
705 return ub->dev_info.flags & UBLK_F_USER_RECOVERY; in ublk_can_use_recovery()
710 struct ublk_device *ub = disk->private_data; in ublk_free_disk() local
712 clear_bit(UB_STATE_USED, &ub->state); in ublk_free_disk()
713 put_device(&ub->cdev_dev); in ublk_free_disk()
730 struct ublk_device *ub = disk->private_data; in ublk_open() local
744 if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV) { in ublk_open()
749 if (curr_uid != ub->dev_info.owner_uid || curr_gid != in ublk_open()
750 ub->dev_info.owner_gid) in ublk_open()
1319 struct ublk_device *ub = driver_data; in ublk_init_hctx() local
1320 struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num); in ublk_init_hctx()
1334 struct ublk_device *ub = container_of(inode->i_cdev, in ublk_ch_open() local
1337 if (test_and_set_bit(UB_STATE_OPEN, &ub->state)) in ublk_ch_open()
1339 filp->private_data = ub; in ublk_ch_open()
1345 struct ublk_device *ub = filp->private_data; in ublk_ch_release() local
1347 clear_bit(UB_STATE_OPEN, &ub->state); in ublk_ch_release()
1354 struct ublk_device *ub = filp->private_data; in ublk_ch_mmap() local
1360 spin_lock(&ub->mm_lock); in ublk_ch_mmap()
1361 if (!ub->mm) in ublk_ch_mmap()
1362 ub->mm = current->mm; in ublk_ch_mmap()
1363 if (current->mm != ub->mm) in ublk_ch_mmap()
1365 spin_unlock(&ub->mm_lock); in ublk_ch_mmap()
1373 end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; in ublk_ch_mmap()
1382 if (sz != ublk_queue_cmd_buf_size(ub, q_id)) in ublk_ch_mmap()
1385 pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT; in ublk_ch_mmap()
1389 static void ublk_commit_completion(struct ublk_device *ub, in ublk_commit_completion() argument
1393 struct ublk_queue *ubq = ublk_get_queue(ub, qid); in ublk_commit_completion()
1402 req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag); in ublk_commit_completion()
1418 static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_abort_queue() argument
1422 if (!ublk_get_device(ub)) in ublk_abort_queue()
1435 rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i); in ublk_abort_queue()
1440 ublk_put_device(ub); in ublk_abort_queue()
1445 struct ublk_device *ub = in ublk_daemon_monitor_work() local
1449 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { in ublk_daemon_monitor_work()
1450 struct ublk_queue *ubq = ublk_get_queue(ub, i); in ublk_daemon_monitor_work()
1454 schedule_work(&ub->quiesce_work); in ublk_daemon_monitor_work()
1456 schedule_work(&ub->stop_work); in ublk_daemon_monitor_work()
1459 ublk_abort_queue(ub, ubq); in ublk_daemon_monitor_work()
1470 if (ub->dev_info.state == UBLK_S_DEV_LIVE) in ublk_daemon_monitor_work()
1471 schedule_delayed_work(&ub->monitor_work, in ublk_daemon_monitor_work()
1505 static void ublk_cancel_dev(struct ublk_device *ub) in ublk_cancel_dev() argument
1509 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_cancel_dev()
1510 ublk_cancel_queue(ublk_get_queue(ub, i)); in ublk_cancel_dev()
1524 static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) in ublk_wait_tagset_rqs_idle() argument
1528 WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue)); in ublk_wait_tagset_rqs_idle()
1531 blk_mq_tagset_busy_iter(&ub->tag_set, in ublk_wait_tagset_rqs_idle()
1539 static void __ublk_quiesce_dev(struct ublk_device *ub) in __ublk_quiesce_dev() argument
1542 __func__, ub->dev_info.dev_id, in __ublk_quiesce_dev()
1543 ub->dev_info.state == UBLK_S_DEV_LIVE ? in __ublk_quiesce_dev()
1545 blk_mq_quiesce_queue(ub->ub_disk->queue); in __ublk_quiesce_dev()
1546 ublk_wait_tagset_rqs_idle(ub); in __ublk_quiesce_dev()
1547 ub->dev_info.state = UBLK_S_DEV_QUIESCED; in __ublk_quiesce_dev()
1548 ublk_cancel_dev(ub); in __ublk_quiesce_dev()
1557 cancel_delayed_work_sync(&ub->monitor_work); in __ublk_quiesce_dev()
1562 struct ublk_device *ub = in ublk_quiesce_work_fn() local
1565 mutex_lock(&ub->mutex); in ublk_quiesce_work_fn()
1566 if (ub->dev_info.state != UBLK_S_DEV_LIVE) in ublk_quiesce_work_fn()
1568 __ublk_quiesce_dev(ub); in ublk_quiesce_work_fn()
1570 mutex_unlock(&ub->mutex); in ublk_quiesce_work_fn()
1573 static void ublk_unquiesce_dev(struct ublk_device *ub) in ublk_unquiesce_dev() argument
1578 __func__, ub->dev_info.dev_id, in ublk_unquiesce_dev()
1579 ub->dev_info.state == UBLK_S_DEV_LIVE ? in ublk_unquiesce_dev()
1586 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_unquiesce_dev()
1587 ublk_get_queue(ub, i)->force_abort = true; in ublk_unquiesce_dev()
1589 blk_mq_unquiesce_queue(ub->ub_disk->queue); in ublk_unquiesce_dev()
1591 blk_mq_kick_requeue_list(ub->ub_disk->queue); in ublk_unquiesce_dev()
1594 static void ublk_stop_dev(struct ublk_device *ub) in ublk_stop_dev() argument
1596 mutex_lock(&ub->mutex); in ublk_stop_dev()
1597 if (ub->dev_info.state == UBLK_S_DEV_DEAD) in ublk_stop_dev()
1599 if (ublk_can_use_recovery(ub)) { in ublk_stop_dev()
1600 if (ub->dev_info.state == UBLK_S_DEV_LIVE) in ublk_stop_dev()
1601 __ublk_quiesce_dev(ub); in ublk_stop_dev()
1602 ublk_unquiesce_dev(ub); in ublk_stop_dev()
1604 del_gendisk(ub->ub_disk); in ublk_stop_dev()
1605 ub->dev_info.state = UBLK_S_DEV_DEAD; in ublk_stop_dev()
1606 ub->dev_info.ublksrv_pid = -1; in ublk_stop_dev()
1607 put_disk(ub->ub_disk); in ublk_stop_dev()
1608 ub->ub_disk = NULL; in ublk_stop_dev()
1610 ublk_cancel_dev(ub); in ublk_stop_dev()
1611 mutex_unlock(&ub->mutex); in ublk_stop_dev()
1612 cancel_delayed_work_sync(&ub->monitor_work); in ublk_stop_dev()
1616 static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_mark_io_ready() argument
1618 mutex_lock(&ub->mutex); in ublk_mark_io_ready()
1623 ub->nr_queues_ready++; in ublk_mark_io_ready()
1626 ub->nr_privileged_daemon++; in ublk_mark_io_ready()
1628 if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues) in ublk_mark_io_ready()
1629 complete_all(&ub->completion); in ublk_mark_io_ready()
1630 mutex_unlock(&ub->mutex); in ublk_mark_io_ready()
1633 static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, in ublk_handle_need_get_data() argument
1636 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_handle_need_get_data()
1637 struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag); in ublk_handle_need_get_data()
1667 struct ublk_device *ub = cmd->file->private_data; in __ublk_ch_uring_cmd() local
1679 if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) in __ublk_ch_uring_cmd()
1682 ubq = ublk_get_queue(ub, ub_cmd->q_id); in __ublk_ch_uring_cmd()
1741 ublk_mark_io_ready(ub, ubq); in __ublk_ch_uring_cmd()
1744 req = blk_mq_tag_to_rq(ub->tag_set.tags[ub_cmd->q_id], tag); in __ublk_ch_uring_cmd()
1767 ublk_commit_completion(ub, ub_cmd); in __ublk_ch_uring_cmd()
1773 ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag); in __ublk_ch_uring_cmd()
1787 static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, in __ublk_check_and_get_req() argument
1795 req = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], tag); in __ublk_check_and_get_req()
1854 struct ublk_device *ub = iocb->ki_filp->private_data; in ublk_check_and_get_req() local
1860 if (!ub) in ublk_check_and_get_req()
1866 if (ub->dev_info.state == UBLK_S_DEV_DEAD) in ublk_check_and_get_req()
1873 if (q_id >= ub->dev_info.nr_hw_queues) in ublk_check_and_get_req()
1876 ubq = ublk_get_queue(ub, q_id); in ublk_check_and_get_req()
1883 req = __ublk_check_and_get_req(ub, ubq, tag, buf_off); in ublk_check_and_get_req()
1947 static void ublk_deinit_queue(struct ublk_device *ub, int q_id) in ublk_deinit_queue() argument
1949 int size = ublk_queue_cmd_buf_size(ub, q_id); in ublk_deinit_queue()
1950 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_deinit_queue()
1958 static int ublk_init_queue(struct ublk_device *ub, int q_id) in ublk_init_queue() argument
1960 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_init_queue()
1965 ubq->flags = ub->dev_info.flags; in ublk_init_queue()
1967 ubq->q_depth = ub->dev_info.queue_depth; in ublk_init_queue()
1968 size = ublk_queue_cmd_buf_size(ub, q_id); in ublk_init_queue()
1975 ubq->dev = ub; in ublk_init_queue()
1979 static void ublk_deinit_queues(struct ublk_device *ub) in ublk_deinit_queues() argument
1981 int nr_queues = ub->dev_info.nr_hw_queues; in ublk_deinit_queues()
1984 if (!ub->__queues) in ublk_deinit_queues()
1988 ublk_deinit_queue(ub, i); in ublk_deinit_queues()
1989 kfree(ub->__queues); in ublk_deinit_queues()
1992 static int ublk_init_queues(struct ublk_device *ub) in ublk_init_queues() argument
1994 int nr_queues = ub->dev_info.nr_hw_queues; in ublk_init_queues()
1995 int depth = ub->dev_info.queue_depth; in ublk_init_queues()
1999 ub->queue_size = ubq_size; in ublk_init_queues()
2000 ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL); in ublk_init_queues()
2001 if (!ub->__queues) in ublk_init_queues()
2005 if (ublk_init_queue(ub, i)) in ublk_init_queues()
2009 init_completion(&ub->completion); in ublk_init_queues()
2013 ublk_deinit_queues(ub); in ublk_init_queues()
2017 static int ublk_alloc_dev_number(struct ublk_device *ub, int idx) in ublk_alloc_dev_number() argument
2025 err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT); in ublk_alloc_dev_number()
2029 err = idr_alloc(&ublk_index_idr, ub, 0, 0, GFP_NOWAIT); in ublk_alloc_dev_number()
2034 ub->ub_number = err; in ublk_alloc_dev_number()
2039 static void ublk_free_dev_number(struct ublk_device *ub) in ublk_free_dev_number() argument
2042 idr_remove(&ublk_index_idr, ub->ub_number); in ublk_free_dev_number()
2049 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); in ublk_cdev_rel() local
2051 blk_mq_free_tag_set(&ub->tag_set); in ublk_cdev_rel()
2052 ublk_deinit_queues(ub); in ublk_cdev_rel()
2053 ublk_free_dev_number(ub); in ublk_cdev_rel()
2054 mutex_destroy(&ub->mutex); in ublk_cdev_rel()
2055 kfree(ub); in ublk_cdev_rel()
2058 static int ublk_add_chdev(struct ublk_device *ub) in ublk_add_chdev() argument
2060 struct device *dev = &ub->cdev_dev; in ublk_add_chdev()
2061 int minor = ub->ub_number; in ublk_add_chdev()
2074 cdev_init(&ub->cdev, &ublk_ch_fops); in ublk_add_chdev()
2075 ret = cdev_device_add(&ub->cdev, dev); in ublk_add_chdev()
2088 struct ublk_device *ub = in ublk_stop_work_fn() local
2091 ublk_stop_dev(ub); in ublk_stop_work_fn()
2095 static void ublk_align_max_io_size(struct ublk_device *ub) in ublk_align_max_io_size() argument
2097 unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes; in ublk_align_max_io_size()
2099 ub->dev_info.max_io_buf_bytes = in ublk_align_max_io_size()
2103 static int ublk_add_tag_set(struct ublk_device *ub) in ublk_add_tag_set() argument
2105 ub->tag_set.ops = &ublk_mq_ops; in ublk_add_tag_set()
2106 ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues; in ublk_add_tag_set()
2107 ub->tag_set.queue_depth = ub->dev_info.queue_depth; in ublk_add_tag_set()
2108 ub->tag_set.numa_node = NUMA_NO_NODE; in ublk_add_tag_set()
2109 ub->tag_set.cmd_size = sizeof(struct ublk_rq_data); in ublk_add_tag_set()
2110 ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; in ublk_add_tag_set()
2111 ub->tag_set.driver_data = ub; in ublk_add_tag_set()
2112 return blk_mq_alloc_tag_set(&ub->tag_set); in ublk_add_tag_set()
2115 static void ublk_remove(struct ublk_device *ub) in ublk_remove() argument
2117 ublk_stop_dev(ub); in ublk_remove()
2118 cancel_work_sync(&ub->stop_work); in ublk_remove()
2119 cancel_work_sync(&ub->quiesce_work); in ublk_remove()
2120 cdev_device_del(&ub->cdev, &ub->cdev_dev); in ublk_remove()
2121 put_device(&ub->cdev_dev); in ublk_remove()
2127 struct ublk_device *ub = NULL; in ublk_get_device_from_id() local
2133 ub = idr_find(&ublk_index_idr, idx); in ublk_get_device_from_id()
2134 if (ub) in ublk_get_device_from_id()
2135 ub = ublk_get_device(ub); in ublk_get_device_from_id()
2138 return ub; in ublk_get_device_from_id()
2141 static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd) in ublk_ctrl_start_dev() argument
2151 if (wait_for_completion_interruptible(&ub->completion) != 0) in ublk_ctrl_start_dev()
2154 schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); in ublk_ctrl_start_dev()
2156 mutex_lock(&ub->mutex); in ublk_ctrl_start_dev()
2157 if (ub->dev_info.state == UBLK_S_DEV_LIVE || in ublk_ctrl_start_dev()
2158 test_bit(UB_STATE_USED, &ub->state)) { in ublk_ctrl_start_dev()
2163 disk = blk_mq_alloc_disk(&ub->tag_set, NULL); in ublk_ctrl_start_dev()
2168 sprintf(disk->disk_name, "ublkb%d", ub->ub_number); in ublk_ctrl_start_dev()
2170 disk->private_data = ub; in ublk_ctrl_start_dev()
2172 ub->dev_info.ublksrv_pid = ublksrv_pid; in ublk_ctrl_start_dev()
2173 ub->ub_disk = disk; in ublk_ctrl_start_dev()
2175 ret = ublk_apply_params(ub); in ublk_ctrl_start_dev()
2180 if (ub->nr_privileged_daemon != ub->nr_queues_ready) in ublk_ctrl_start_dev()
2183 get_device(&ub->cdev_dev); in ublk_ctrl_start_dev()
2184 ub->dev_info.state = UBLK_S_DEV_LIVE; in ublk_ctrl_start_dev()
2186 if (ublk_dev_is_zoned(ub)) { in ublk_ctrl_start_dev()
2187 ret = ublk_revalidate_disk_zones(ub); in ublk_ctrl_start_dev()
2196 set_bit(UB_STATE_USED, &ub->state); in ublk_ctrl_start_dev()
2200 ub->dev_info.state = UBLK_S_DEV_DEAD; in ublk_ctrl_start_dev()
2201 ublk_put_device(ub); in ublk_ctrl_start_dev()
2207 mutex_unlock(&ub->mutex); in ublk_ctrl_start_dev()
2211 static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub, in ublk_ctrl_get_queue_affinity() argument
2230 if (queue >= ub->dev_info.nr_hw_queues) in ublk_ctrl_get_queue_affinity()
2237 if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue) in ublk_ctrl_get_queue_affinity()
2268 struct ublk_device *ub; in ublk_ctrl_add_dev() local
2319 ub = kzalloc(sizeof(*ub), GFP_KERNEL); in ublk_ctrl_add_dev()
2320 if (!ub) in ublk_ctrl_add_dev()
2322 mutex_init(&ub->mutex); in ublk_ctrl_add_dev()
2323 spin_lock_init(&ub->mm_lock); in ublk_ctrl_add_dev()
2324 INIT_WORK(&ub->quiesce_work, ublk_quiesce_work_fn); in ublk_ctrl_add_dev()
2325 INIT_WORK(&ub->stop_work, ublk_stop_work_fn); in ublk_ctrl_add_dev()
2326 INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work); in ublk_ctrl_add_dev()
2328 ret = ublk_alloc_dev_number(ub, header->dev_id); in ublk_ctrl_add_dev()
2332 memcpy(&ub->dev_info, &info, sizeof(info)); in ublk_ctrl_add_dev()
2335 ub->dev_info.dev_id = ub->ub_number; in ublk_ctrl_add_dev()
2343 ub->dev_info.flags &= UBLK_F_ALL; in ublk_ctrl_add_dev()
2345 ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | in ublk_ctrl_add_dev()
2349 if (ublk_dev_is_user_copy(ub)) in ublk_ctrl_add_dev()
2350 ub->dev_info.flags &= ~UBLK_F_NEED_GET_DATA; in ublk_ctrl_add_dev()
2353 if (ublk_dev_is_zoned(ub) && in ublk_ctrl_add_dev()
2354 (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || !ublk_dev_is_user_copy(ub))) { in ublk_ctrl_add_dev()
2360 ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY; in ublk_ctrl_add_dev()
2362 ub->dev_info.nr_hw_queues = min_t(unsigned int, in ublk_ctrl_add_dev()
2363 ub->dev_info.nr_hw_queues, nr_cpu_ids); in ublk_ctrl_add_dev()
2364 ublk_align_max_io_size(ub); in ublk_ctrl_add_dev()
2366 ret = ublk_init_queues(ub); in ublk_ctrl_add_dev()
2370 ret = ublk_add_tag_set(ub); in ublk_ctrl_add_dev()
2375 if (copy_to_user(argp, &ub->dev_info, sizeof(info))) in ublk_ctrl_add_dev()
2382 ret = ublk_add_chdev(ub); in ublk_ctrl_add_dev()
2386 blk_mq_free_tag_set(&ub->tag_set); in ublk_ctrl_add_dev()
2388 ublk_deinit_queues(ub); in ublk_ctrl_add_dev()
2390 ublk_free_dev_number(ub); in ublk_ctrl_add_dev()
2392 mutex_destroy(&ub->mutex); in ublk_ctrl_add_dev()
2393 kfree(ub); in ublk_ctrl_add_dev()
2412 struct ublk_device *ub = *p_ub; in ublk_ctrl_del_dev() local
2413 int idx = ub->ub_number; in ublk_ctrl_del_dev()
2420 if (!test_bit(UB_STATE_DELETED, &ub->state)) { in ublk_ctrl_del_dev()
2421 ublk_remove(ub); in ublk_ctrl_del_dev()
2422 set_bit(UB_STATE_DELETED, &ub->state); in ublk_ctrl_del_dev()
2427 ublk_put_device(ub); in ublk_ctrl_del_dev()
2458 static int ublk_ctrl_stop_dev(struct ublk_device *ub) in ublk_ctrl_stop_dev() argument
2460 ublk_stop_dev(ub); in ublk_ctrl_stop_dev()
2461 cancel_work_sync(&ub->stop_work); in ublk_ctrl_stop_dev()
2462 cancel_work_sync(&ub->quiesce_work); in ublk_ctrl_stop_dev()
2467 static int ublk_ctrl_get_dev_info(struct ublk_device *ub, in ublk_ctrl_get_dev_info() argument
2476 if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info))) in ublk_ctrl_get_dev_info()
2483 static void ublk_ctrl_fill_params_devt(struct ublk_device *ub) in ublk_ctrl_fill_params_devt() argument
2485 ub->params.devt.char_major = MAJOR(ub->cdev_dev.devt); in ublk_ctrl_fill_params_devt()
2486 ub->params.devt.char_minor = MINOR(ub->cdev_dev.devt); in ublk_ctrl_fill_params_devt()
2488 if (ub->ub_disk) { in ublk_ctrl_fill_params_devt()
2489 ub->params.devt.disk_major = MAJOR(disk_devt(ub->ub_disk)); in ublk_ctrl_fill_params_devt()
2490 ub->params.devt.disk_minor = MINOR(disk_devt(ub->ub_disk)); in ublk_ctrl_fill_params_devt()
2492 ub->params.devt.disk_major = 0; in ublk_ctrl_fill_params_devt()
2493 ub->params.devt.disk_minor = 0; in ublk_ctrl_fill_params_devt()
2495 ub->params.types |= UBLK_PARAM_TYPE_DEVT; in ublk_ctrl_fill_params_devt()
2498 static int ublk_ctrl_get_params(struct ublk_device *ub, in ublk_ctrl_get_params() argument
2518 mutex_lock(&ub->mutex); in ublk_ctrl_get_params()
2519 ublk_ctrl_fill_params_devt(ub); in ublk_ctrl_get_params()
2520 if (copy_to_user(argp, &ub->params, ph.len)) in ublk_ctrl_get_params()
2524 mutex_unlock(&ub->mutex); in ublk_ctrl_get_params()
2529 static int ublk_ctrl_set_params(struct ublk_device *ub, in ublk_ctrl_set_params() argument
2550 mutex_lock(&ub->mutex); in ublk_ctrl_set_params()
2551 if (ub->dev_info.state == UBLK_S_DEV_LIVE) { in ublk_ctrl_set_params()
2553 } else if (copy_from_user(&ub->params, argp, ph.len)) { in ublk_ctrl_set_params()
2557 ub->params.types &= UBLK_PARAM_TYPE_ALL; in ublk_ctrl_set_params()
2558 ret = ublk_validate_params(ub); in ublk_ctrl_set_params()
2560 ub->params.types = 0; in ublk_ctrl_set_params()
2562 mutex_unlock(&ub->mutex); in ublk_ctrl_set_params()
2567 static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_queue_reinit() argument
2590 static int ublk_ctrl_start_recovery(struct ublk_device *ub, in ublk_ctrl_start_recovery() argument
2597 mutex_lock(&ub->mutex); in ublk_ctrl_start_recovery()
2598 if (!ublk_can_use_recovery(ub)) in ublk_ctrl_start_recovery()
2613 if (test_bit(UB_STATE_OPEN, &ub->state) || in ublk_ctrl_start_recovery()
2614 ub->dev_info.state != UBLK_S_DEV_QUIESCED) { in ublk_ctrl_start_recovery()
2619 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_ctrl_start_recovery()
2620 ublk_queue_reinit(ub, ublk_get_queue(ub, i)); in ublk_ctrl_start_recovery()
2622 ub->mm = NULL; in ublk_ctrl_start_recovery()
2623 ub->nr_queues_ready = 0; in ublk_ctrl_start_recovery()
2624 ub->nr_privileged_daemon = 0; in ublk_ctrl_start_recovery()
2625 init_completion(&ub->completion); in ublk_ctrl_start_recovery()
2628 mutex_unlock(&ub->mutex); in ublk_ctrl_start_recovery()
2632 static int ublk_ctrl_end_recovery(struct ublk_device *ub, in ublk_ctrl_end_recovery() argument
2640 __func__, ub->dev_info.nr_hw_queues, header->dev_id); in ublk_ctrl_end_recovery()
2642 if (wait_for_completion_interruptible(&ub->completion)) in ublk_ctrl_end_recovery()
2646 __func__, ub->dev_info.nr_hw_queues, header->dev_id); in ublk_ctrl_end_recovery()
2648 mutex_lock(&ub->mutex); in ublk_ctrl_end_recovery()
2649 if (!ublk_can_use_recovery(ub)) in ublk_ctrl_end_recovery()
2652 if (ub->dev_info.state != UBLK_S_DEV_QUIESCED) { in ublk_ctrl_end_recovery()
2656 ub->dev_info.ublksrv_pid = ublksrv_pid; in ublk_ctrl_end_recovery()
2659 blk_mq_unquiesce_queue(ub->ub_disk->queue); in ublk_ctrl_end_recovery()
2662 blk_mq_kick_requeue_list(ub->ub_disk->queue); in ublk_ctrl_end_recovery()
2663 ub->dev_info.state = UBLK_S_DEV_LIVE; in ublk_ctrl_end_recovery()
2664 schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); in ublk_ctrl_end_recovery()
2667 mutex_unlock(&ub->mutex); in ublk_ctrl_end_recovery()
2690 static int ublk_char_dev_permission(struct ublk_device *ub, in ublk_char_dev_permission() argument
2706 if (stat.rdev != ub->cdev_dev.devt || !S_ISCHR(stat.mode)) in ublk_char_dev_permission()
2716 static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, in ublk_ctrl_uring_cmd_permission() argument
2720 bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV; in ublk_ctrl_uring_cmd_permission()
2777 ret = ublk_char_dev_permission(ub, dev_path, mask); in ublk_ctrl_uring_cmd_permission()
2783 __func__, ub->ub_number, cmd->cmd_op, in ublk_ctrl_uring_cmd_permission()
2784 ub->dev_info.owner_uid, ub->dev_info.owner_gid, in ublk_ctrl_uring_cmd_permission()
2795 struct ublk_device *ub = NULL; in ublk_ctrl_uring_cmd() local
2818 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_uring_cmd()
2819 if (!ub) in ublk_ctrl_uring_cmd()
2822 ret = ublk_ctrl_uring_cmd_permission(ub, cmd); in ublk_ctrl_uring_cmd()
2829 ret = ublk_ctrl_start_dev(ub, cmd); in ublk_ctrl_uring_cmd()
2832 ret = ublk_ctrl_stop_dev(ub); in ublk_ctrl_uring_cmd()
2836 ret = ublk_ctrl_get_dev_info(ub, cmd); in ublk_ctrl_uring_cmd()
2842 ret = ublk_ctrl_del_dev(&ub); in ublk_ctrl_uring_cmd()
2845 ret = ublk_ctrl_get_queue_affinity(ub, cmd); in ublk_ctrl_uring_cmd()
2848 ret = ublk_ctrl_get_params(ub, cmd); in ublk_ctrl_uring_cmd()
2851 ret = ublk_ctrl_set_params(ub, cmd); in ublk_ctrl_uring_cmd()
2854 ret = ublk_ctrl_start_recovery(ub, cmd); in ublk_ctrl_uring_cmd()
2857 ret = ublk_ctrl_end_recovery(ub, cmd); in ublk_ctrl_uring_cmd()
2865 if (ub) in ublk_ctrl_uring_cmd()
2866 ublk_put_device(ub); in ublk_ctrl_uring_cmd()
2919 struct ublk_device *ub; in ublk_exit() local
2922 idr_for_each_entry(&ublk_index_idr, ub, id) in ublk_exit()
2923 ublk_remove(ub); in ublk_exit()