Lines Matching refs:ub
190 static void ublk_dev_param_basic_apply(struct ublk_device *ub) in ublk_dev_param_basic_apply() argument
192 struct request_queue *q = ub->ub_disk->queue; in ublk_dev_param_basic_apply()
193 const struct ublk_param_basic *p = &ub->params.basic; in ublk_dev_param_basic_apply()
212 set_disk_ro(ub->ub_disk, true); in ublk_dev_param_basic_apply()
214 set_capacity(ub->ub_disk, p->dev_sectors); in ublk_dev_param_basic_apply()
217 static void ublk_dev_param_discard_apply(struct ublk_device *ub) in ublk_dev_param_discard_apply() argument
219 struct request_queue *q = ub->ub_disk->queue; in ublk_dev_param_discard_apply()
220 const struct ublk_param_discard *p = &ub->params.discard; in ublk_dev_param_discard_apply()
230 static int ublk_validate_params(const struct ublk_device *ub) in ublk_validate_params() argument
233 if (ub->params.types & UBLK_PARAM_TYPE_BASIC) { in ublk_validate_params()
234 const struct ublk_param_basic *p = &ub->params.basic; in ublk_validate_params()
242 if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9)) in ublk_validate_params()
247 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) { in ublk_validate_params()
248 const struct ublk_param_discard *p = &ub->params.discard; in ublk_validate_params()
261 static int ublk_apply_params(struct ublk_device *ub) in ublk_apply_params() argument
263 if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC)) in ublk_apply_params()
266 ublk_dev_param_basic_apply(ub); in ublk_apply_params()
268 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) in ublk_apply_params()
269 ublk_dev_param_discard_apply(ub); in ublk_apply_params()
289 static struct ublk_device *ublk_get_device(struct ublk_device *ub) in ublk_get_device() argument
291 if (kobject_get_unless_zero(&ub->cdev_dev.kobj)) in ublk_get_device()
292 return ub; in ublk_get_device()
296 static void ublk_put_device(struct ublk_device *ub) in ublk_put_device() argument
298 put_device(&ub->cdev_dev); in ublk_put_device()
319 static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id) in ublk_queue_cmd_buf() argument
321 return ublk_get_queue(ub, q_id)->io_cmd_buf; in ublk_queue_cmd_buf()
324 static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id) in ublk_queue_cmd_buf_size() argument
326 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_queue_cmd_buf_size()
349 static inline bool ublk_can_use_recovery(struct ublk_device *ub) in ublk_can_use_recovery() argument
351 if (ub->dev_info.flags & UBLK_F_USER_RECOVERY) in ublk_can_use_recovery()
358 struct ublk_device *ub = disk->private_data; in ublk_free_disk() local
360 clear_bit(UB_STATE_USED, &ub->state); in ublk_free_disk()
361 put_device(&ub->cdev_dev); in ublk_free_disk()
879 struct ublk_device *ub = driver_data; in ublk_init_hctx() local
880 struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num); in ublk_init_hctx()
903 struct ublk_device *ub = container_of(inode->i_cdev, in ublk_ch_open() local
906 if (test_and_set_bit(UB_STATE_OPEN, &ub->state)) in ublk_ch_open()
908 filp->private_data = ub; in ublk_ch_open()
914 struct ublk_device *ub = filp->private_data; in ublk_ch_release() local
916 clear_bit(UB_STATE_OPEN, &ub->state); in ublk_ch_release()
923 struct ublk_device *ub = filp->private_data; in ublk_ch_mmap() local
929 spin_lock(&ub->mm_lock); in ublk_ch_mmap()
930 if (!ub->mm) in ublk_ch_mmap()
931 ub->mm = current->mm; in ublk_ch_mmap()
932 if (current->mm != ub->mm) in ublk_ch_mmap()
934 spin_unlock(&ub->mm_lock); in ublk_ch_mmap()
942 end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; in ublk_ch_mmap()
951 if (sz != ublk_queue_cmd_buf_size(ub, q_id)) in ublk_ch_mmap()
954 pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT; in ublk_ch_mmap()
958 static void ublk_commit_completion(struct ublk_device *ub, in ublk_commit_completion() argument
962 struct ublk_queue *ubq = ublk_get_queue(ub, qid); in ublk_commit_completion()
971 req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag); in ublk_commit_completion()
982 static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_abort_queue() argument
986 if (!ublk_get_device(ub)) in ublk_abort_queue()
999 rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i); in ublk_abort_queue()
1004 ublk_put_device(ub); in ublk_abort_queue()
1009 struct ublk_device *ub = in ublk_daemon_monitor_work() local
1013 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { in ublk_daemon_monitor_work()
1014 struct ublk_queue *ubq = ublk_get_queue(ub, i); in ublk_daemon_monitor_work()
1018 schedule_work(&ub->quiesce_work); in ublk_daemon_monitor_work()
1020 schedule_work(&ub->stop_work); in ublk_daemon_monitor_work()
1023 ublk_abort_queue(ub, ubq); in ublk_daemon_monitor_work()
1034 if (ub->dev_info.state == UBLK_S_DEV_LIVE) in ublk_daemon_monitor_work()
1035 schedule_delayed_work(&ub->monitor_work, in ublk_daemon_monitor_work()
1063 static void ublk_cancel_dev(struct ublk_device *ub) in ublk_cancel_dev() argument
1067 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_cancel_dev()
1068 ublk_cancel_queue(ublk_get_queue(ub, i)); in ublk_cancel_dev()
1082 static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) in ublk_wait_tagset_rqs_idle() argument
1086 WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue)); in ublk_wait_tagset_rqs_idle()
1089 blk_mq_tagset_busy_iter(&ub->tag_set, in ublk_wait_tagset_rqs_idle()
1097 static void __ublk_quiesce_dev(struct ublk_device *ub) in __ublk_quiesce_dev() argument
1100 __func__, ub->dev_info.dev_id, in __ublk_quiesce_dev()
1101 ub->dev_info.state == UBLK_S_DEV_LIVE ? in __ublk_quiesce_dev()
1103 blk_mq_quiesce_queue(ub->ub_disk->queue); in __ublk_quiesce_dev()
1104 ublk_wait_tagset_rqs_idle(ub); in __ublk_quiesce_dev()
1105 ub->dev_info.state = UBLK_S_DEV_QUIESCED; in __ublk_quiesce_dev()
1106 ublk_cancel_dev(ub); in __ublk_quiesce_dev()
1115 cancel_delayed_work_sync(&ub->monitor_work); in __ublk_quiesce_dev()
1120 struct ublk_device *ub = in ublk_quiesce_work_fn() local
1123 mutex_lock(&ub->mutex); in ublk_quiesce_work_fn()
1124 if (ub->dev_info.state != UBLK_S_DEV_LIVE) in ublk_quiesce_work_fn()
1126 __ublk_quiesce_dev(ub); in ublk_quiesce_work_fn()
1128 mutex_unlock(&ub->mutex); in ublk_quiesce_work_fn()
1131 static void ublk_unquiesce_dev(struct ublk_device *ub) in ublk_unquiesce_dev() argument
1136 __func__, ub->dev_info.dev_id, in ublk_unquiesce_dev()
1137 ub->dev_info.state == UBLK_S_DEV_LIVE ? in ublk_unquiesce_dev()
1144 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_unquiesce_dev()
1145 ublk_get_queue(ub, i)->force_abort = true; in ublk_unquiesce_dev()
1147 blk_mq_unquiesce_queue(ub->ub_disk->queue); in ublk_unquiesce_dev()
1149 blk_mq_kick_requeue_list(ub->ub_disk->queue); in ublk_unquiesce_dev()
1152 static void ublk_stop_dev(struct ublk_device *ub) in ublk_stop_dev() argument
1154 mutex_lock(&ub->mutex); in ublk_stop_dev()
1155 if (ub->dev_info.state == UBLK_S_DEV_DEAD) in ublk_stop_dev()
1157 if (ublk_can_use_recovery(ub)) { in ublk_stop_dev()
1158 if (ub->dev_info.state == UBLK_S_DEV_LIVE) in ublk_stop_dev()
1159 __ublk_quiesce_dev(ub); in ublk_stop_dev()
1160 ublk_unquiesce_dev(ub); in ublk_stop_dev()
1162 del_gendisk(ub->ub_disk); in ublk_stop_dev()
1163 ub->dev_info.state = UBLK_S_DEV_DEAD; in ublk_stop_dev()
1164 ub->dev_info.ublksrv_pid = -1; in ublk_stop_dev()
1165 put_disk(ub->ub_disk); in ublk_stop_dev()
1166 ub->ub_disk = NULL; in ublk_stop_dev()
1168 ublk_cancel_dev(ub); in ublk_stop_dev()
1169 mutex_unlock(&ub->mutex); in ublk_stop_dev()
1170 cancel_delayed_work_sync(&ub->monitor_work); in ublk_stop_dev()
1174 static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_mark_io_ready() argument
1176 mutex_lock(&ub->mutex); in ublk_mark_io_ready()
1181 ub->nr_queues_ready++; in ublk_mark_io_ready()
1183 if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues) in ublk_mark_io_ready()
1184 complete_all(&ub->completion); in ublk_mark_io_ready()
1185 mutex_unlock(&ub->mutex); in ublk_mark_io_ready()
1188 static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, in ublk_handle_need_get_data() argument
1191 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_handle_need_get_data()
1192 struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag); in ublk_handle_need_get_data()
1200 struct ublk_device *ub = cmd->file->private_data; in ublk_ch_uring_cmd() local
1214 if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) in ublk_ch_uring_cmd()
1217 ubq = ublk_get_queue(ub, ub_cmd->q_id); in ublk_ch_uring_cmd()
1263 ublk_mark_io_ready(ub, ubq); in ublk_ch_uring_cmd()
1274 ublk_commit_completion(ub, ub_cmd); in ublk_ch_uring_cmd()
1282 ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag); in ublk_ch_uring_cmd()
1305 static void ublk_deinit_queue(struct ublk_device *ub, int q_id) in ublk_deinit_queue() argument
1307 int size = ublk_queue_cmd_buf_size(ub, q_id); in ublk_deinit_queue()
1308 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_deinit_queue()
1316 static int ublk_init_queue(struct ublk_device *ub, int q_id) in ublk_init_queue() argument
1318 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); in ublk_init_queue()
1323 ubq->flags = ub->dev_info.flags; in ublk_init_queue()
1325 ubq->q_depth = ub->dev_info.queue_depth; in ublk_init_queue()
1326 size = ublk_queue_cmd_buf_size(ub, q_id); in ublk_init_queue()
1333 ubq->dev = ub; in ublk_init_queue()
1337 static void ublk_deinit_queues(struct ublk_device *ub) in ublk_deinit_queues() argument
1339 int nr_queues = ub->dev_info.nr_hw_queues; in ublk_deinit_queues()
1342 if (!ub->__queues) in ublk_deinit_queues()
1346 ublk_deinit_queue(ub, i); in ublk_deinit_queues()
1347 kfree(ub->__queues); in ublk_deinit_queues()
1350 static int ublk_init_queues(struct ublk_device *ub) in ublk_init_queues() argument
1352 int nr_queues = ub->dev_info.nr_hw_queues; in ublk_init_queues()
1353 int depth = ub->dev_info.queue_depth; in ublk_init_queues()
1357 ub->queue_size = ubq_size; in ublk_init_queues()
1358 ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL); in ublk_init_queues()
1359 if (!ub->__queues) in ublk_init_queues()
1363 if (ublk_init_queue(ub, i)) in ublk_init_queues()
1367 init_completion(&ub->completion); in ublk_init_queues()
1371 ublk_deinit_queues(ub); in ublk_init_queues()
1375 static int ublk_alloc_dev_number(struct ublk_device *ub, int idx) in ublk_alloc_dev_number() argument
1383 err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT); in ublk_alloc_dev_number()
1387 err = idr_alloc(&ublk_index_idr, ub, 0, 0, GFP_NOWAIT); in ublk_alloc_dev_number()
1392 ub->ub_number = err; in ublk_alloc_dev_number()
1397 static void ublk_free_dev_number(struct ublk_device *ub) in ublk_free_dev_number() argument
1400 idr_remove(&ublk_index_idr, ub->ub_number); in ublk_free_dev_number()
1407 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); in ublk_cdev_rel() local
1409 blk_mq_free_tag_set(&ub->tag_set); in ublk_cdev_rel()
1410 ublk_deinit_queues(ub); in ublk_cdev_rel()
1411 ublk_free_dev_number(ub); in ublk_cdev_rel()
1412 mutex_destroy(&ub->mutex); in ublk_cdev_rel()
1413 kfree(ub); in ublk_cdev_rel()
1416 static int ublk_add_chdev(struct ublk_device *ub) in ublk_add_chdev() argument
1418 struct device *dev = &ub->cdev_dev; in ublk_add_chdev()
1419 int minor = ub->ub_number; in ublk_add_chdev()
1432 cdev_init(&ub->cdev, &ublk_ch_fops); in ublk_add_chdev()
1433 ret = cdev_device_add(&ub->cdev, dev); in ublk_add_chdev()
1444 struct ublk_device *ub = in ublk_stop_work_fn() local
1447 ublk_stop_dev(ub); in ublk_stop_work_fn()
1451 static void ublk_align_max_io_size(struct ublk_device *ub) in ublk_align_max_io_size() argument
1453 unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes; in ublk_align_max_io_size()
1455 ub->dev_info.max_io_buf_bytes = in ublk_align_max_io_size()
1459 static int ublk_add_tag_set(struct ublk_device *ub) in ublk_add_tag_set() argument
1461 ub->tag_set.ops = &ublk_mq_ops; in ublk_add_tag_set()
1462 ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues; in ublk_add_tag_set()
1463 ub->tag_set.queue_depth = ub->dev_info.queue_depth; in ublk_add_tag_set()
1464 ub->tag_set.numa_node = NUMA_NO_NODE; in ublk_add_tag_set()
1465 ub->tag_set.cmd_size = sizeof(struct ublk_rq_data); in ublk_add_tag_set()
1466 ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; in ublk_add_tag_set()
1467 ub->tag_set.driver_data = ub; in ublk_add_tag_set()
1468 return blk_mq_alloc_tag_set(&ub->tag_set); in ublk_add_tag_set()
1471 static void ublk_remove(struct ublk_device *ub) in ublk_remove() argument
1473 ublk_stop_dev(ub); in ublk_remove()
1474 cancel_work_sync(&ub->stop_work); in ublk_remove()
1475 cancel_work_sync(&ub->quiesce_work); in ublk_remove()
1476 cdev_device_del(&ub->cdev, &ub->cdev_dev); in ublk_remove()
1477 put_device(&ub->cdev_dev); in ublk_remove()
1482 struct ublk_device *ub = NULL; in ublk_get_device_from_id() local
1488 ub = idr_find(&ublk_index_idr, idx); in ublk_get_device_from_id()
1489 if (ub) in ublk_get_device_from_id()
1490 ub = ublk_get_device(ub); in ublk_get_device_from_id()
1493 return ub; in ublk_get_device_from_id()
1500 struct ublk_device *ub; in ublk_ctrl_start_dev() local
1507 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_start_dev()
1508 if (!ub) in ublk_ctrl_start_dev()
1511 wait_for_completion_interruptible(&ub->completion); in ublk_ctrl_start_dev()
1513 schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); in ublk_ctrl_start_dev()
1515 mutex_lock(&ub->mutex); in ublk_ctrl_start_dev()
1516 if (ub->dev_info.state == UBLK_S_DEV_LIVE || in ublk_ctrl_start_dev()
1517 test_bit(UB_STATE_USED, &ub->state)) { in ublk_ctrl_start_dev()
1522 disk = blk_mq_alloc_disk(&ub->tag_set, ub); in ublk_ctrl_start_dev()
1527 sprintf(disk->disk_name, "ublkb%d", ub->ub_number); in ublk_ctrl_start_dev()
1529 disk->private_data = ub; in ublk_ctrl_start_dev()
1531 ub->dev_info.ublksrv_pid = ublksrv_pid; in ublk_ctrl_start_dev()
1532 ub->ub_disk = disk; in ublk_ctrl_start_dev()
1534 ret = ublk_apply_params(ub); in ublk_ctrl_start_dev()
1538 get_device(&ub->cdev_dev); in ublk_ctrl_start_dev()
1545 ublk_put_device(ub); in ublk_ctrl_start_dev()
1548 set_bit(UB_STATE_USED, &ub->state); in ublk_ctrl_start_dev()
1549 ub->dev_info.state = UBLK_S_DEV_LIVE; in ublk_ctrl_start_dev()
1554 mutex_unlock(&ub->mutex); in ublk_ctrl_start_dev()
1555 ublk_put_device(ub); in ublk_ctrl_start_dev()
1563 struct ublk_device *ub; in ublk_ctrl_get_queue_affinity() local
1577 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_get_queue_affinity()
1578 if (!ub) in ublk_ctrl_get_queue_affinity()
1582 if (queue >= ub->dev_info.nr_hw_queues) in ublk_ctrl_get_queue_affinity()
1590 if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue) in ublk_ctrl_get_queue_affinity()
1606 ublk_put_device(ub); in ublk_ctrl_get_queue_affinity()
1623 struct ublk_device *ub; in ublk_ctrl_add_dev() local
1647 ub = kzalloc(sizeof(*ub), GFP_KERNEL); in ublk_ctrl_add_dev()
1648 if (!ub) in ublk_ctrl_add_dev()
1650 mutex_init(&ub->mutex); in ublk_ctrl_add_dev()
1651 spin_lock_init(&ub->mm_lock); in ublk_ctrl_add_dev()
1652 INIT_WORK(&ub->quiesce_work, ublk_quiesce_work_fn); in ublk_ctrl_add_dev()
1653 INIT_WORK(&ub->stop_work, ublk_stop_work_fn); in ublk_ctrl_add_dev()
1654 INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work); in ublk_ctrl_add_dev()
1656 ret = ublk_alloc_dev_number(ub, header->dev_id); in ublk_ctrl_add_dev()
1660 memcpy(&ub->dev_info, &info, sizeof(info)); in ublk_ctrl_add_dev()
1663 ub->dev_info.dev_id = ub->ub_number; in ublk_ctrl_add_dev()
1671 ub->dev_info.flags &= UBLK_F_ALL; in ublk_ctrl_add_dev()
1674 ub->dev_info.flags |= UBLK_F_URING_CMD_COMP_IN_TASK; in ublk_ctrl_add_dev()
1677 ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY; in ublk_ctrl_add_dev()
1679 ub->dev_info.nr_hw_queues = min_t(unsigned int, in ublk_ctrl_add_dev()
1680 ub->dev_info.nr_hw_queues, nr_cpu_ids); in ublk_ctrl_add_dev()
1681 ublk_align_max_io_size(ub); in ublk_ctrl_add_dev()
1683 ret = ublk_init_queues(ub); in ublk_ctrl_add_dev()
1687 ret = ublk_add_tag_set(ub); in ublk_ctrl_add_dev()
1692 if (copy_to_user(argp, &ub->dev_info, sizeof(info))) in ublk_ctrl_add_dev()
1699 ret = ublk_add_chdev(ub); in ublk_ctrl_add_dev()
1703 blk_mq_free_tag_set(&ub->tag_set); in ublk_ctrl_add_dev()
1705 ublk_deinit_queues(ub); in ublk_ctrl_add_dev()
1707 ublk_free_dev_number(ub); in ublk_ctrl_add_dev()
1709 mutex_destroy(&ub->mutex); in ublk_ctrl_add_dev()
1710 kfree(ub); in ublk_ctrl_add_dev()
1729 struct ublk_device *ub; in ublk_ctrl_del_dev() local
1736 ub = ublk_get_device_from_id(idx); in ublk_ctrl_del_dev()
1737 if (ub) { in ublk_ctrl_del_dev()
1738 ublk_remove(ub); in ublk_ctrl_del_dev()
1739 ublk_put_device(ub); in ublk_ctrl_del_dev()
1768 struct ublk_device *ub; in ublk_ctrl_stop_dev() local
1770 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_stop_dev()
1771 if (!ub) in ublk_ctrl_stop_dev()
1774 ublk_stop_dev(ub); in ublk_ctrl_stop_dev()
1775 cancel_work_sync(&ub->stop_work); in ublk_ctrl_stop_dev()
1776 cancel_work_sync(&ub->quiesce_work); in ublk_ctrl_stop_dev()
1778 ublk_put_device(ub); in ublk_ctrl_stop_dev()
1786 struct ublk_device *ub; in ublk_ctrl_get_dev_info() local
1792 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_get_dev_info()
1793 if (!ub) in ublk_ctrl_get_dev_info()
1796 if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info))) in ublk_ctrl_get_dev_info()
1798 ublk_put_device(ub); in ublk_ctrl_get_dev_info()
1808 struct ublk_device *ub; in ublk_ctrl_get_params() local
1823 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_get_params()
1824 if (!ub) in ublk_ctrl_get_params()
1827 mutex_lock(&ub->mutex); in ublk_ctrl_get_params()
1828 if (copy_to_user(argp, &ub->params, ph.len)) in ublk_ctrl_get_params()
1832 mutex_unlock(&ub->mutex); in ublk_ctrl_get_params()
1834 ublk_put_device(ub); in ublk_ctrl_get_params()
1843 struct ublk_device *ub; in ublk_ctrl_set_params() local
1858 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_set_params()
1859 if (!ub) in ublk_ctrl_set_params()
1863 mutex_lock(&ub->mutex); in ublk_ctrl_set_params()
1864 if (ub->dev_info.state == UBLK_S_DEV_LIVE) { in ublk_ctrl_set_params()
1866 } else if (copy_from_user(&ub->params, argp, ph.len)) { in ublk_ctrl_set_params()
1870 ub->params.types &= UBLK_PARAM_TYPE_ALL; in ublk_ctrl_set_params()
1871 ret = ublk_validate_params(ub); in ublk_ctrl_set_params()
1873 mutex_unlock(&ub->mutex); in ublk_ctrl_set_params()
1874 ublk_put_device(ub); in ublk_ctrl_set_params()
1879 static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) in ublk_queue_reinit() argument
1904 struct ublk_device *ub; in ublk_ctrl_start_recovery() local
1908 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_start_recovery()
1909 if (!ub) in ublk_ctrl_start_recovery()
1912 mutex_lock(&ub->mutex); in ublk_ctrl_start_recovery()
1913 if (!ublk_can_use_recovery(ub)) in ublk_ctrl_start_recovery()
1928 if (test_bit(UB_STATE_OPEN, &ub->state) || in ublk_ctrl_start_recovery()
1929 ub->dev_info.state != UBLK_S_DEV_QUIESCED) { in ublk_ctrl_start_recovery()
1934 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) in ublk_ctrl_start_recovery()
1935 ublk_queue_reinit(ub, ublk_get_queue(ub, i)); in ublk_ctrl_start_recovery()
1937 ub->mm = NULL; in ublk_ctrl_start_recovery()
1938 ub->nr_queues_ready = 0; in ublk_ctrl_start_recovery()
1939 init_completion(&ub->completion); in ublk_ctrl_start_recovery()
1942 mutex_unlock(&ub->mutex); in ublk_ctrl_start_recovery()
1943 ublk_put_device(ub); in ublk_ctrl_start_recovery()
1951 struct ublk_device *ub; in ublk_ctrl_end_recovery() local
1954 ub = ublk_get_device_from_id(header->dev_id); in ublk_ctrl_end_recovery()
1955 if (!ub) in ublk_ctrl_end_recovery()
1959 __func__, ub->dev_info.nr_hw_queues, header->dev_id); in ublk_ctrl_end_recovery()
1961 wait_for_completion_interruptible(&ub->completion); in ublk_ctrl_end_recovery()
1963 __func__, ub->dev_info.nr_hw_queues, header->dev_id); in ublk_ctrl_end_recovery()
1965 mutex_lock(&ub->mutex); in ublk_ctrl_end_recovery()
1966 if (!ublk_can_use_recovery(ub)) in ublk_ctrl_end_recovery()
1969 if (ub->dev_info.state != UBLK_S_DEV_QUIESCED) { in ublk_ctrl_end_recovery()
1973 ub->dev_info.ublksrv_pid = ublksrv_pid; in ublk_ctrl_end_recovery()
1976 blk_mq_unquiesce_queue(ub->ub_disk->queue); in ublk_ctrl_end_recovery()
1979 blk_mq_kick_requeue_list(ub->ub_disk->queue); in ublk_ctrl_end_recovery()
1980 ub->dev_info.state = UBLK_S_DEV_LIVE; in ublk_ctrl_end_recovery()
1981 schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); in ublk_ctrl_end_recovery()
1984 mutex_unlock(&ub->mutex); in ublk_ctrl_end_recovery()
1985 ublk_put_device(ub); in ublk_ctrl_end_recovery()
2089 struct ublk_device *ub; in ublk_exit() local
2096 idr_for_each_entry(&ublk_index_idr, ub, id) in ublk_exit()
2097 ublk_remove(ub); in ublk_exit()