25 	struct kiocb			kiocb;  member
82 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
104 		rw->kiocb.ki_ioprio = ioprio;  in io_prep_rw()
106 		rw->kiocb.ki_ioprio = get_current_ioprio();  in io_prep_rw()
108 	rw->kiocb.dio_complete = NULL;  in io_prep_rw()
133 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)  in io_rw_done()  argument
150 		kiocb->ki_complete(kiocb, ret);  in io_rw_done()
158 	if (rw->kiocb.ki_pos != -1)  in io_kiocb_update_pos()
159 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
163 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
164 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
167 	rw->kiocb.ki_pos = 0;  in io_kiocb_update_pos()
229 		kiocb_end_write(&rw->kiocb);  in io_req_end_write()
241 	if (rw->kiocb.ki_flags & IOCB_WRITE) {  in io_req_io_end()
285 	struct kiocb *kiocb = &rw->kiocb;  in io_req_rw_complete()  local
287 	if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) {  in io_req_rw_complete()
288 		long res = kiocb->dio_complete(rw->kiocb.private);  in io_req_rw_complete()
303 static void io_complete_rw(struct kiocb *kiocb, long res)  in io_complete_rw()  argument
305 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw()
308 	if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) {  in io_complete_rw()
317 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)  in io_complete_rw_iopoll()  argument
319 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw_iopoll()
322 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_iopoll()
343 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
344 	if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {  in kiocb_done()
356 		io_rw_done(&rw->kiocb, ret);  in kiocb_done()
427 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)  in io_kiocb_ppos()  argument
429 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;  in io_kiocb_ppos()
438 	struct kiocb *kiocb = &rw->kiocb;  in loop_rw_iter()  local
439 	struct file *file = kiocb->ki_filp;  in loop_rw_iter()
448 	if (kiocb->ki_flags & IOCB_HIPRI)  in loop_rw_iter()
450 	if ((kiocb->ki_flags & IOCB_NOWAIT) &&  in loop_rw_iter()
451 	    !(kiocb->ki_filp->f_flags & O_NONBLOCK))  in loop_rw_iter()
454 	ppos = io_kiocb_ppos(kiocb);  in loop_rw_iter()
601 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
624 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_should_retry()  local
631 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))  in io_rw_should_retry()
645 	kiocb->ki_flags |= IOCB_WAITQ;  in io_rw_should_retry()
646 	kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_rw_should_retry()
647 	kiocb->ki_waitq = wait;  in io_rw_should_retry()
653 	struct file *file = rw->kiocb.ki_filp;  in io_iter_do_read()
656 		return call_read_iter(file, &rw->kiocb, iter);  in io_iter_do_read()
672 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_init_file()  local
683 	kiocb->ki_flags = file->f_iocb_flags;  in io_rw_init_file()
684 	ret = kiocb_set_rw_flags(kiocb, rw->flags);  in io_rw_init_file()
687 	kiocb->ki_flags |= IOCB_ALLOC_CACHE;  in io_rw_init_file()
694 	if ((kiocb->ki_flags & IOCB_NOWAIT) ||  in io_rw_init_file()
699 		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)  in io_rw_init_file()
702 		kiocb->private = NULL;  in io_rw_init_file()
703 		kiocb->ki_flags |= IOCB_HIPRI;  in io_rw_init_file()
704 		kiocb->ki_complete = io_complete_rw_iopoll;  in io_rw_init_file()
707 		if (kiocb->ki_flags & IOCB_HIPRI)  in io_rw_init_file()
709 		kiocb->ki_complete = io_complete_rw;  in io_rw_init_file()
720 	struct kiocb *kiocb = &rw->kiocb;  in io_read()  local
765 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_read()
768 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_read()
838 			kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
853 		kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
868 	struct kiocb *kiocb = &rw->kiocb;  in io_write()  local
897 		if (!(kiocb->ki_flags & IOCB_DIRECT) &&  in io_write()
898 			!(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) &&  in io_write()
902 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_write()
905 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_write()
917 		kiocb_start_write(kiocb);  in io_write()
918 	kiocb->ki_flags |= IOCB_WRITE;  in io_write()
921 		ret2 = call_write_iter(req->file, kiocb, &s->iter);  in io_write()
936 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))  in io_write()
949 			trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,  in io_write()
964 			if (kiocb->ki_flags & IOCB_WRITE)  in io_write()
975 			if (kiocb->ki_flags & IOCB_WRITE)  in io_write()
1031 			ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);  in io_do_iopoll()