591 void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)  in kiocb_set_cancel_fn()  argument
593 	struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw);  in kiocb_set_cancel_fn()
601 	if (!(iocb->ki_flags & IOCB_AIO_RW))  in kiocb_set_cancel_fn()
1106 static inline void iocb_destroy(struct aio_kiocb *iocb)  in iocb_destroy()  argument
1108 	if (iocb->ki_eventfd)  in iocb_destroy()
1109 		eventfd_ctx_put(iocb->ki_eventfd);  in iocb_destroy()
1110 	if (iocb->ki_filp)  in iocb_destroy()
1111 		fput(iocb->ki_filp);  in iocb_destroy()
1112 	percpu_ref_put(&iocb->ki_ctx->reqs);  in iocb_destroy()
1113 	kmem_cache_free(kiocb_cachep, iocb);  in iocb_destroy()
1119 static void aio_complete(struct aio_kiocb *iocb)  in aio_complete()  argument
1121 	struct kioctx	*ctx = iocb->ki_ctx;  in aio_complete()
1143 	*event = iocb->ki_res;  in aio_complete()
1147 	pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb,  in aio_complete()
1148 		 (void __user *)(unsigned long)iocb->ki_res.obj,  in aio_complete()
1149 		 iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2);  in aio_complete()
1168 	pr_debug("added to ring %p at [%u]\n", iocb, tail);  in aio_complete()
1175 	if (iocb->ki_eventfd)  in aio_complete()
1176 		eventfd_signal(iocb->ki_eventfd, 1);  in aio_complete()
1190 static inline void iocb_put(struct aio_kiocb *iocb)  in iocb_put()  argument
1192 	if (refcount_dec_and_test(&iocb->ki_refcnt)) {  in iocb_put()
1193 		aio_complete(iocb);  in iocb_put()
1194 		iocb_destroy(iocb);  in iocb_put()
1437 static void aio_remove_iocb(struct aio_kiocb *iocb)  in aio_remove_iocb()  argument
1439 	struct kioctx *ctx = iocb->ki_ctx;  in aio_remove_iocb()
1443 	list_del(&iocb->ki_list);  in aio_remove_iocb()
1449 	struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);  in aio_complete_rw()  local
1451 	if (!list_empty_careful(&iocb->ki_list))  in aio_complete_rw()
1452 		aio_remove_iocb(iocb);  in aio_complete_rw()
1461 	iocb->ki_res.res = res;  in aio_complete_rw()
1462 	iocb->ki_res.res2 = 0;  in aio_complete_rw()
1463 	iocb_put(iocb);  in aio_complete_rw()
1466 static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb)  in aio_prep_rw()  argument
1472 	req->ki_pos = iocb->aio_offset;  in aio_prep_rw()
1474 	if (iocb->aio_flags & IOCB_FLAG_RESFD)  in aio_prep_rw()
1476 	if (iocb->aio_flags & IOCB_FLAG_IOPRIO) {  in aio_prep_rw()
1482 		ret = ioprio_check_cap(iocb->aio_reqprio);  in aio_prep_rw()
1488 		req->ki_ioprio = iocb->aio_reqprio;  in aio_prep_rw()
1492 	ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);  in aio_prep_rw()
1500 static ssize_t aio_setup_rw(int rw, const struct iocb *iocb,  in aio_setup_rw()  argument
1504 	void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;  in aio_setup_rw()
1505 	size_t len = iocb->aio_nbytes;  in aio_setup_rw()
1536 static int aio_read(struct kiocb *req, const struct iocb *iocb,  in aio_read()  argument
1544 	ret = aio_prep_rw(req, iocb);  in aio_read()
1553 	ret = aio_setup_rw(ITER_DEST, iocb, &iovec, vectored, compat, &iter);  in aio_read()
1563 static int aio_write(struct kiocb *req, const struct iocb *iocb,  in aio_write()  argument
1571 	ret = aio_prep_rw(req, iocb);  in aio_write()
1581 	ret = aio_setup_rw(ITER_SOURCE, iocb, &iovec, vectored, compat, &iter);  in aio_write()
1597 	struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work);  in aio_fsync_work()  local
1598 	const struct cred *old_cred = override_creds(iocb->fsync.creds);  in aio_fsync_work()
1600 	iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync);  in aio_fsync_work()
1602 	put_cred(iocb->fsync.creds);  in aio_fsync_work()
1603 	iocb_put(iocb);  in aio_fsync_work()
1606 static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb,  in aio_fsync()  argument
1609 	if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes ||  in aio_fsync()
1610 			iocb->aio_rw_flags))  in aio_fsync()
1629 	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);  in aio_poll_put_work()  local
1631 	iocb_put(iocb);  in aio_poll_put_work()
1682 	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);  in aio_poll_complete_work()  local
1684 	struct kioctx *ctx = iocb->ki_ctx;  in aio_poll_complete_work()
1717 	list_del_init(&iocb->ki_list);  in aio_poll_complete_work()
1718 	iocb->ki_res.res = mangle_poll(mask);  in aio_poll_complete_work()
1721 	iocb_put(iocb);  in aio_poll_complete_work()
1725 static int aio_poll_cancel(struct kiocb *iocb)  in aio_poll_cancel()  argument
1727 	struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);  in aio_poll_cancel()
1746 	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);  in aio_poll_wake()  local
1768 	    spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) {  in aio_poll_wake()
1769 		struct kioctx *ctx = iocb->ki_ctx;  in aio_poll_wake()
1772 		list_del(&iocb->ki_list);  in aio_poll_wake()
1773 		iocb->ki_res.res = mangle_poll(mask);  in aio_poll_wake()
1774 		if (iocb->ki_eventfd && !eventfd_signal_allowed()) {  in aio_poll_wake()
1775 			iocb = NULL;  in aio_poll_wake()
1780 		if (iocb)  in aio_poll_wake()
1781 			iocb_put(iocb);  in aio_poll_wake()
1826 	struct aio_kiocb		*iocb;  member
1845 	pt->iocb->poll.head = head;  in aio_poll_queue_proc()
1846 	add_wait_queue(head, &pt->iocb->poll.wait);  in aio_poll_queue_proc()
1849 static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)  in aio_poll()  argument
1858 	if ((u16)iocb->aio_buf != iocb->aio_buf)  in aio_poll()
1861 	if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)  in aio_poll()
1865 	req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;  in aio_poll()
1874 	apt.iocb = aiocb;  in aio_poll()
1924 static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb,  in __io_submit_one()  argument
1925 			   struct iocb __user *user_iocb, struct aio_kiocb *req,  in __io_submit_one()
1928 	req->ki_filp = fget(iocb->aio_fildes);  in __io_submit_one()
1932 	if (iocb->aio_flags & IOCB_FLAG_RESFD) {  in __io_submit_one()
1940 		eventfd = eventfd_ctx_fdget(iocb->aio_resfd);  in __io_submit_one()
1953 	req->ki_res.data = iocb->aio_data;  in __io_submit_one()
1957 	switch (iocb->aio_lio_opcode) {  in __io_submit_one()
1959 		return aio_read(&req->rw, iocb, false, compat);  in __io_submit_one()
1961 		return aio_write(&req->rw, iocb, false, compat);  in __io_submit_one()
1963 		return aio_read(&req->rw, iocb, true, compat);  in __io_submit_one()
1965 		return aio_write(&req->rw, iocb, true, compat);  in __io_submit_one()
1967 		return aio_fsync(&req->fsync, iocb, false);  in __io_submit_one()
1969 		return aio_fsync(&req->fsync, iocb, true);  in __io_submit_one()
1971 		return aio_poll(req, iocb);  in __io_submit_one()
1973 		pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode);  in __io_submit_one()
1978 static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,  in io_submit_one()
1982 	struct iocb iocb;  in io_submit_one()  local
1985 	if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb))))  in io_submit_one()
1989 	if (unlikely(iocb.aio_reserved2)) {  in io_submit_one()
1996 	    (iocb.aio_buf != (unsigned long)iocb.aio_buf) ||  in io_submit_one()
1997 	    (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) ||  in io_submit_one()
1998 	    ((ssize_t)iocb.aio_nbytes < 0)  in io_submit_one()
2008 	err = __io_submit_one(ctx, &iocb, user_iocb, req, compat);  in io_submit_one()
2038 		struct iocb __user * __user *, iocbpp)  in SYSCALL_DEFINE3()
2060 		struct iocb __user *user_iocb;  in SYSCALL_DEFINE3()
2131 SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,  in SYSCALL_DEFINE3()  argument
2138 	u64 obj = (u64)(unsigned long)iocb;  in SYSCALL_DEFINE3()
2140 	if (unlikely(get_user(key, &iocb->aio_key)))  in SYSCALL_DEFINE3()