41 void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
45 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
46 int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
48 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
49 int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
54 unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
56 void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
60 static inline void io_kbuf_recycle_ring(struct io_kiocb *req)  in io_kbuf_recycle_ring()  argument
69 	if (req->buf_list) {  in io_kbuf_recycle_ring()
70 		if (req->flags & REQ_F_PARTIAL_IO) {  in io_kbuf_recycle_ring()
78 			req->buf_list->head++;  in io_kbuf_recycle_ring()
79 			req->buf_list = NULL;  in io_kbuf_recycle_ring()
81 			req->buf_index = req->buf_list->bgid;  in io_kbuf_recycle_ring()
82 			req->flags &= ~REQ_F_BUFFER_RING;  in io_kbuf_recycle_ring()
87 static inline bool io_do_buffer_select(struct io_kiocb *req)  in io_do_buffer_select()  argument
89 	if (!(req->flags & REQ_F_BUFFER_SELECT))  in io_do_buffer_select()
91 	return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));  in io_do_buffer_select()
94 static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)  in io_kbuf_recycle()  argument
96 	if (req->flags & REQ_F_BUFFER_SELECTED)  in io_kbuf_recycle()
97 		io_kbuf_recycle_legacy(req, issue_flags);  in io_kbuf_recycle()
98 	if (req->flags & REQ_F_BUFFER_RING)  in io_kbuf_recycle()
99 		io_kbuf_recycle_ring(req);  in io_kbuf_recycle()
102 static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,  in __io_put_kbuf_list()  argument
105 	unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);  in __io_put_kbuf_list()
107 	if (req->flags & REQ_F_BUFFER_RING) {  in __io_put_kbuf_list()
108 		if (req->buf_list) {  in __io_put_kbuf_list()
109 			req->buf_index = req->buf_list->bgid;  in __io_put_kbuf_list()
110 			req->buf_list->head++;  in __io_put_kbuf_list()
112 		req->flags &= ~REQ_F_BUFFER_RING;  in __io_put_kbuf_list()
114 		req->buf_index = req->kbuf->bgid;  in __io_put_kbuf_list()
115 		list_add(&req->kbuf->list, list);  in __io_put_kbuf_list()
116 		req->flags &= ~REQ_F_BUFFER_SELECTED;  in __io_put_kbuf_list()
122 static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)  in io_put_kbuf_comp()  argument
124 	lockdep_assert_held(&req->ctx->completion_lock);  in io_put_kbuf_comp()
126 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))  in io_put_kbuf_comp()
128 	return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp);  in io_put_kbuf_comp()
131 static inline unsigned int io_put_kbuf(struct io_kiocb *req,  in io_put_kbuf()  argument
135 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))  in io_put_kbuf()
137 	return __io_put_kbuf(req, issue_flags);  in io_put_kbuf()