189 	struct iocb __user	*ki_user_iocb;	/* user's aiocb */
546 void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)  in kiocb_set_cancel_fn()  argument
548 	struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw);  in kiocb_set_cancel_fn()
1051 static inline void iocb_put(struct aio_kiocb *iocb)  in iocb_put()  argument
1053 	if (refcount_read(&iocb->ki_refcnt) == 0 ||  in iocb_put()
1054 	    refcount_dec_and_test(&iocb->ki_refcnt)) {  in iocb_put()
1055 		percpu_ref_put(&iocb->ki_ctx->reqs);  in iocb_put()
1056 		kmem_cache_free(kiocb_cachep, iocb);  in iocb_put()
1063 static void aio_complete(struct aio_kiocb *iocb, long res, long res2)  in aio_complete()  argument
1065 	struct kioctx	*ctx = iocb->ki_ctx;  in aio_complete()
1087 	event->obj = (u64)(unsigned long)iocb->ki_user_iocb;  in aio_complete()
1088 	event->data = iocb->ki_user_data;  in aio_complete()
1096 		 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data,  in aio_complete()
1117 	pr_debug("added to ring %p at [%u]\n", iocb, tail);  in aio_complete()
1124 	if (iocb->ki_eventfd) {  in aio_complete()
1125 		eventfd_signal(iocb->ki_eventfd, 1);  in aio_complete()
1126 		eventfd_ctx_put(iocb->ki_eventfd);  in aio_complete()
1139 	iocb_put(iocb);  in aio_complete()
1384 static void aio_remove_iocb(struct aio_kiocb *iocb)  in aio_remove_iocb()  argument
1386 	struct kioctx *ctx = iocb->ki_ctx;  in aio_remove_iocb()
1390 	list_del(&iocb->ki_list);  in aio_remove_iocb()
1396 	struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);  in aio_complete_rw()  local
1398 	if (!list_empty_careful(&iocb->ki_list))  in aio_complete_rw()
1399 		aio_remove_iocb(iocb);  in aio_complete_rw()
1414 	aio_complete(iocb, res, res2);  in aio_complete_rw()
1417 static int aio_prep_rw(struct kiocb *req, struct iocb *iocb)  in aio_prep_rw()  argument
1421 	req->ki_filp = fget(iocb->aio_fildes);  in aio_prep_rw()
1425 	req->ki_pos = iocb->aio_offset;  in aio_prep_rw()
1427 	if (iocb->aio_flags & IOCB_FLAG_RESFD)  in aio_prep_rw()
1430 	if (iocb->aio_flags & IOCB_FLAG_IOPRIO) {  in aio_prep_rw()
1436 		ret = ioprio_check_cap(iocb->aio_reqprio);  in aio_prep_rw()
1442 		req->ki_ioprio = iocb->aio_reqprio;  in aio_prep_rw()
1446 	ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);  in aio_prep_rw()
1452 static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec,  in aio_setup_rw()  argument
1455 	void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;  in aio_setup_rw()
1456 	size_t len = iocb->aio_nbytes;  in aio_setup_rw()
1491 static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored,  in aio_read()  argument
1499 	ret = aio_prep_rw(req, iocb);  in aio_read()
1511 	ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);  in aio_read()
1524 static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored,  in aio_write()  argument
1532 	ret = aio_prep_rw(req, iocb);  in aio_write()
1544 	ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);  in aio_write()
1580 static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync)  in aio_fsync()  argument
1582 	if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes ||  in aio_fsync()
1583 			iocb->aio_rw_flags))  in aio_fsync()
1586 	req->file = fget(iocb->aio_fildes);  in aio_fsync()
1600 static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask)  in aio_poll_complete()  argument
1602 	struct file *file = iocb->poll.file;  in aio_poll_complete()
1604 	aio_complete(iocb, mangle_poll(mask), 0);  in aio_poll_complete()
1611 	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);  in aio_poll_complete_work()  local
1613 	struct kioctx *ctx = iocb->ki_ctx;  in aio_poll_complete_work()
1632 	list_del_init(&iocb->ki_list);  in aio_poll_complete_work()
1635 	aio_poll_complete(iocb, mask);  in aio_poll_complete_work()
1639 static int aio_poll_cancel(struct kiocb *iocb)  in aio_poll_cancel()  argument
1641 	struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);  in aio_poll_cancel()
1659 	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);  in aio_poll_wake()  local
1670 		if (spin_trylock(&iocb->ki_ctx->ctx_lock)) {  in aio_poll_wake()
1671 			list_del(&iocb->ki_list);  in aio_poll_wake()
1672 			spin_unlock(&iocb->ki_ctx->ctx_lock);  in aio_poll_wake()
1675 			aio_poll_complete(iocb, mask);  in aio_poll_wake()
1687 	struct aio_kiocb		*iocb;  member
1698 	if (unlikely(pt->iocb->poll.head)) {  in aio_poll_queue_proc()
1704 	pt->iocb->poll.head = head;  in aio_poll_queue_proc()
1705 	add_wait_queue(head, &pt->iocb->poll.wait);  in aio_poll_queue_proc()
1708 static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb)  in aio_poll()  argument
1716 	if ((u16)iocb->aio_buf != iocb->aio_buf)  in aio_poll()
1719 	if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)  in aio_poll()
1723 	req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;  in aio_poll()
1724 	req->file = fget(iocb->aio_fildes);  in aio_poll()
1730 	apt.iocb = aiocb;  in aio_poll()
1776 static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,  in io_submit_one()
1780 	struct iocb iocb;  in io_submit_one()  local
1783 	if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb))))  in io_submit_one()
1787 	if (unlikely(iocb.aio_reserved2)) {  in io_submit_one()
1794 	    (iocb.aio_buf != (unsigned long)iocb.aio_buf) ||  in io_submit_one()
1795 	    (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) ||  in io_submit_one()
1796 	    ((ssize_t)iocb.aio_nbytes < 0)  in io_submit_one()
1806 	if (iocb.aio_flags & IOCB_FLAG_RESFD) {  in io_submit_one()
1813 		req->ki_eventfd = eventfd_ctx_fdget((int) iocb.aio_resfd);  in io_submit_one()
1828 	req->ki_user_data = iocb.aio_data;  in io_submit_one()
1830 	switch (iocb.aio_lio_opcode) {  in io_submit_one()
1832 		ret = aio_read(&req->rw, &iocb, false, compat);  in io_submit_one()
1835 		ret = aio_write(&req->rw, &iocb, false, compat);  in io_submit_one()
1838 		ret = aio_read(&req->rw, &iocb, true, compat);  in io_submit_one()
1841 		ret = aio_write(&req->rw, &iocb, true, compat);  in io_submit_one()
1844 		ret = aio_fsync(&req->fsync, &iocb, false);  in io_submit_one()
1847 		ret = aio_fsync(&req->fsync, &iocb, true);  in io_submit_one()
1850 		ret = aio_poll(req, &iocb);  in io_submit_one()
1853 		pr_debug("invalid aio operation %d\n", iocb.aio_lio_opcode);  in io_submit_one()
1888 		struct iocb __user * __user *, iocbpp)  in SYSCALL_DEFINE3()
1909 		struct iocb __user *user_iocb;  in SYSCALL_DEFINE3()
1971 lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb)  in lookup_kiocb()  argument
1979 		if (kiocb->ki_user_iocb == iocb)  in lookup_kiocb()
1995 SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,  in SYSCALL_DEFINE3()  argument
2003 	if (unlikely(get_user(key, &iocb->aio_key)))  in SYSCALL_DEFINE3()
2013 	kiocb = lookup_kiocb(ctx, iocb);  in SYSCALL_DEFINE3()