Lines Matching refs:sqd
1080 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
7309 static inline bool io_sqd_events_pending(struct io_sq_data *sqd) in io_sqd_events_pending() argument
7311 return READ_ONCE(sqd->state); in io_sqd_events_pending()
7370 static void io_sqd_update_thread_idle(struct io_sq_data *sqd) in io_sqd_update_thread_idle() argument
7375 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sqd_update_thread_idle()
7377 sqd->sq_thread_idle = sq_thread_idle; in io_sqd_update_thread_idle()
7380 static bool io_sqd_handle_event(struct io_sq_data *sqd) in io_sqd_handle_event() argument
7385 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || in io_sqd_handle_event()
7387 mutex_unlock(&sqd->lock); in io_sqd_handle_event()
7391 mutex_lock(&sqd->lock); in io_sqd_handle_event()
7393 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); in io_sqd_handle_event()
7398 struct io_sq_data *sqd = data; in io_sq_thread() local
7404 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid); in io_sq_thread()
7407 if (sqd->sq_cpu != -1) in io_sq_thread()
7408 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu)); in io_sq_thread()
7413 mutex_lock(&sqd->lock); in io_sq_thread()
7417 if (io_sqd_events_pending(sqd) || signal_pending(current)) { in io_sq_thread()
7418 if (io_sqd_handle_event(sqd)) in io_sq_thread()
7420 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
7423 cap_entries = !list_is_singular(&sqd->ctx_list); in io_sq_thread()
7424 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { in io_sq_thread()
7436 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
7440 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); in io_sq_thread()
7441 if (!io_sqd_events_pending(sqd) && !current->task_works) { in io_sq_thread()
7444 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { in io_sq_thread()
7459 mutex_unlock(&sqd->lock); in io_sq_thread()
7461 mutex_lock(&sqd->lock); in io_sq_thread()
7463 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sq_thread()
7467 finish_wait(&sqd->wait, &wait); in io_sq_thread()
7468 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
7471 io_uring_cancel_generic(true, sqd); in io_sq_thread()
7472 sqd->thread = NULL; in io_sq_thread()
7473 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sq_thread()
7476 mutex_unlock(&sqd->lock); in io_sq_thread()
7478 complete(&sqd->exited); in io_sq_thread()
7878 static void io_sq_thread_unpark(struct io_sq_data *sqd) in io_sq_thread_unpark() argument
7879 __releases(&sqd->lock) in io_sq_thread_unpark()
7881 WARN_ON_ONCE(sqd->thread == current); in io_sq_thread_unpark()
7887 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_unpark()
7888 if (atomic_dec_return(&sqd->park_pending)) in io_sq_thread_unpark()
7889 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_unpark()
7890 mutex_unlock(&sqd->lock); in io_sq_thread_unpark()
7893 static void io_sq_thread_park(struct io_sq_data *sqd) in io_sq_thread_park() argument
7894 __acquires(&sqd->lock) in io_sq_thread_park()
7896 WARN_ON_ONCE(sqd->thread == current); in io_sq_thread_park()
7898 atomic_inc(&sqd->park_pending); in io_sq_thread_park()
7899 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_park()
7900 mutex_lock(&sqd->lock); in io_sq_thread_park()
7901 if (sqd->thread) in io_sq_thread_park()
7902 wake_up_process(sqd->thread); in io_sq_thread_park()
7905 static void io_sq_thread_stop(struct io_sq_data *sqd) in io_sq_thread_stop() argument
7907 WARN_ON_ONCE(sqd->thread == current); in io_sq_thread_stop()
7908 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)); in io_sq_thread_stop()
7910 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); in io_sq_thread_stop()
7911 mutex_lock(&sqd->lock); in io_sq_thread_stop()
7912 if (sqd->thread) in io_sq_thread_stop()
7913 wake_up_process(sqd->thread); in io_sq_thread_stop()
7914 mutex_unlock(&sqd->lock); in io_sq_thread_stop()
7915 wait_for_completion(&sqd->exited); in io_sq_thread_stop()
7918 static void io_put_sq_data(struct io_sq_data *sqd) in io_put_sq_data() argument
7920 if (refcount_dec_and_test(&sqd->refs)) { in io_put_sq_data()
7921 WARN_ON_ONCE(atomic_read(&sqd->park_pending)); in io_put_sq_data()
7923 io_sq_thread_stop(sqd); in io_put_sq_data()
7924 kfree(sqd); in io_put_sq_data()
7930 struct io_sq_data *sqd = ctx->sq_data; in io_sq_thread_finish() local
7932 if (sqd) { in io_sq_thread_finish()
7933 io_sq_thread_park(sqd); in io_sq_thread_finish()
7935 io_sqd_update_thread_idle(sqd); in io_sq_thread_finish()
7936 io_sq_thread_unpark(sqd); in io_sq_thread_finish()
7938 io_put_sq_data(sqd); in io_sq_thread_finish()
7946 struct io_sq_data *sqd; in io_attach_sq_data() local
7958 sqd = ctx_attach->sq_data; in io_attach_sq_data()
7959 if (!sqd) { in io_attach_sq_data()
7963 if (sqd->task_tgid != current->tgid) { in io_attach_sq_data()
7968 refcount_inc(&sqd->refs); in io_attach_sq_data()
7970 return sqd; in io_attach_sq_data()
7976 struct io_sq_data *sqd; in io_get_sq_data() local
7980 sqd = io_attach_sq_data(p); in io_get_sq_data()
7981 if (!IS_ERR(sqd)) { in io_get_sq_data()
7983 return sqd; in io_get_sq_data()
7986 if (PTR_ERR(sqd) != -EPERM) in io_get_sq_data()
7987 return sqd; in io_get_sq_data()
7990 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); in io_get_sq_data()
7991 if (!sqd) in io_get_sq_data()
7994 atomic_set(&sqd->park_pending, 0); in io_get_sq_data()
7995 refcount_set(&sqd->refs, 1); in io_get_sq_data()
7996 INIT_LIST_HEAD(&sqd->ctx_list); in io_get_sq_data()
7997 mutex_init(&sqd->lock); in io_get_sq_data()
7998 init_waitqueue_head(&sqd->wait); in io_get_sq_data()
7999 init_completion(&sqd->exited); in io_get_sq_data()
8000 return sqd; in io_get_sq_data()
8627 struct io_sq_data *sqd; in io_sq_offload_create() local
8630 sqd = io_get_sq_data(p, &attached); in io_sq_offload_create()
8631 if (IS_ERR(sqd)) { in io_sq_offload_create()
8632 ret = PTR_ERR(sqd); in io_sq_offload_create()
8637 ctx->sq_data = sqd; in io_sq_offload_create()
8642 io_sq_thread_park(sqd); in io_sq_offload_create()
8643 list_add(&ctx->sqd_list, &sqd->ctx_list); in io_sq_offload_create()
8644 io_sqd_update_thread_idle(sqd); in io_sq_offload_create()
8646 ret = (attached && !sqd->thread) ? -ENXIO : 0; in io_sq_offload_create()
8647 io_sq_thread_unpark(sqd); in io_sq_offload_create()
8660 sqd->sq_cpu = cpu; in io_sq_offload_create()
8662 sqd->sq_cpu = -1; in io_sq_offload_create()
8665 sqd->task_pid = current->pid; in io_sq_offload_create()
8666 sqd->task_tgid = current->tgid; in io_sq_offload_create()
8667 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); in io_sq_offload_create()
8673 sqd->thread = tsk; in io_sq_offload_create()
9399 struct io_sq_data *sqd = ctx->sq_data; in io_ring_exit_work() local
9402 io_sq_thread_park(sqd); in io_ring_exit_work()
9403 tsk = sqd->thread; in io_ring_exit_work()
9407 io_sq_thread_unpark(sqd); in io_ring_exit_work()
9752 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) in io_uring_cancel_generic() argument
9759 WARN_ON_ONCE(sqd && sqd->thread != current); in io_uring_cancel_generic()
9774 if (!sqd) { in io_uring_cancel_generic()
9786 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_uring_cancel_generic()
10656 struct io_sq_data *sqd = NULL; in io_register_iowq_max_workers() local
10667 sqd = ctx->sq_data; in io_register_iowq_max_workers()
10668 if (sqd) { in io_register_iowq_max_workers()
10674 refcount_inc(&sqd->refs); in io_register_iowq_max_workers()
10676 mutex_lock(&sqd->lock); in io_register_iowq_max_workers()
10678 if (sqd->thread) in io_register_iowq_max_workers()
10679 tctx = sqd->thread->io_uring; in io_register_iowq_max_workers()
10699 if (sqd) { in io_register_iowq_max_workers()
10700 mutex_unlock(&sqd->lock); in io_register_iowq_max_workers()
10701 io_put_sq_data(sqd); in io_register_iowq_max_workers()
10708 if (sqd) in io_register_iowq_max_workers()
10725 if (sqd) { in io_register_iowq_max_workers()
10726 mutex_unlock(&sqd->lock); in io_register_iowq_max_workers()
10727 io_put_sq_data(sqd); in io_register_iowq_max_workers()