Lines Matching refs:req
34 struct io_kiocb *req; member
68 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) in io_poll_get_ownership_slowpath() argument
77 v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_get_ownership_slowpath()
80 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership_slowpath()
89 static inline bool io_poll_get_ownership(struct io_kiocb *req) in io_poll_get_ownership() argument
91 if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) in io_poll_get_ownership()
92 return io_poll_get_ownership_slowpath(req); in io_poll_get_ownership()
93 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership()
96 static void io_poll_mark_cancelled(struct io_kiocb *req) in io_poll_mark_cancelled() argument
98 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); in io_poll_mark_cancelled()
101 static struct io_poll *io_poll_get_double(struct io_kiocb *req) in io_poll_get_double() argument
104 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_double()
105 return req->async_data; in io_poll_get_double()
106 return req->apoll->double_poll; in io_poll_get_double()
109 static struct io_poll *io_poll_get_single(struct io_kiocb *req) in io_poll_get_single() argument
111 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_single()
112 return io_kiocb_to_cmd(req, struct io_poll); in io_poll_get_single()
113 return &req->apoll->poll; in io_poll_get_single()
116 static void io_poll_req_insert(struct io_kiocb *req) in io_poll_req_insert() argument
118 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_insert()
119 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert()
123 hlist_add_head(&req->hash_node, &hb->list); in io_poll_req_insert()
127 static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_poll_req_delete() argument
129 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_delete()
130 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_delete()
134 hash_del(&req->hash_node); in io_poll_req_delete()
138 static void io_poll_req_insert_locked(struct io_kiocb *req) in io_poll_req_insert_locked() argument
140 struct io_hash_table *table = &req->ctx->cancel_table_locked; in io_poll_req_insert_locked()
141 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert_locked()
143 lockdep_assert_held(&req->ctx->uring_lock); in io_poll_req_insert_locked()
145 hlist_add_head(&req->hash_node, &table->hbs[index].list); in io_poll_req_insert_locked()
148 static void io_poll_tw_hash_eject(struct io_kiocb *req, bool *locked) in io_poll_tw_hash_eject() argument
150 struct io_ring_ctx *ctx = req->ctx; in io_poll_tw_hash_eject()
152 if (req->flags & REQ_F_HASH_LOCKED) { in io_poll_tw_hash_eject()
160 hash_del(&req->hash_node); in io_poll_tw_hash_eject()
161 req->flags &= ~REQ_F_HASH_LOCKED; in io_poll_tw_hash_eject()
163 io_poll_req_delete(req, ctx); in io_poll_tw_hash_eject()
190 static void io_poll_remove_entries(struct io_kiocb *req) in io_poll_remove_entries() argument
196 if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL))) in io_poll_remove_entries()
215 if (req->flags & REQ_F_SINGLE_POLL) in io_poll_remove_entries()
216 io_poll_remove_entry(io_poll_get_single(req)); in io_poll_remove_entries()
217 if (req->flags & REQ_F_DOUBLE_POLL) in io_poll_remove_entries()
218 io_poll_remove_entry(io_poll_get_double(req)); in io_poll_remove_entries()
239 static int io_poll_check_events(struct io_kiocb *req, bool *locked) in io_poll_check_events() argument
241 struct io_ring_ctx *ctx = req->ctx; in io_poll_check_events()
245 if (unlikely(req->task->flags & PF_EXITING)) in io_poll_check_events()
249 v = atomic_read(&req->poll_refs); in io_poll_check_events()
262 req->cqe.res = 0; in io_poll_check_events()
264 req->cqe.res = 0; in io_poll_check_events()
270 atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_check_events()
275 if (!req->cqe.res) { in io_poll_check_events()
276 struct poll_table_struct pt = { ._key = req->apoll_events }; in io_poll_check_events()
277 req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events; in io_poll_check_events()
284 if (unlikely(!req->cqe.res)) { in io_poll_check_events()
286 if (!(req->apoll_events & EPOLLONESHOT)) in io_poll_check_events()
291 if (req->apoll_events & EPOLLONESHOT) in io_poll_check_events()
293 if (io_is_uring_fops(req->file)) in io_poll_check_events()
297 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { in io_poll_check_events()
298 __poll_t mask = mangle_poll(req->cqe.res & in io_poll_check_events()
299 req->apoll_events); in io_poll_check_events()
301 if (!io_post_aux_cqe(ctx, req->cqe.user_data, in io_poll_check_events()
303 io_req_set_res(req, mask, 0); in io_poll_check_events()
307 int ret = io_poll_issue(req, locked); in io_poll_check_events()
315 req->cqe.res = 0; in io_poll_check_events()
321 } while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) & in io_poll_check_events()
327 static void io_poll_task_func(struct io_kiocb *req, bool *locked) in io_poll_task_func() argument
331 ret = io_poll_check_events(req, locked); in io_poll_task_func()
336 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_task_func()
337 req->cqe.res = mangle_poll(req->cqe.res & poll->events); in io_poll_task_func()
339 io_poll_remove_entries(req); in io_poll_task_func()
340 io_poll_tw_hash_eject(req, locked); in io_poll_task_func()
341 io_req_task_submit(req, locked); in io_poll_task_func()
344 req->cqe.res = ret; in io_poll_task_func()
345 req_set_fail(req); in io_poll_task_func()
348 io_poll_remove_entries(req); in io_poll_task_func()
349 io_poll_tw_hash_eject(req, locked); in io_poll_task_func()
351 io_req_set_res(req, req->cqe.res, 0); in io_poll_task_func()
352 io_req_task_complete(req, locked); in io_poll_task_func()
355 static void io_apoll_task_func(struct io_kiocb *req, bool *locked) in io_apoll_task_func() argument
359 ret = io_poll_check_events(req, locked); in io_apoll_task_func()
363 io_poll_remove_entries(req); in io_apoll_task_func()
364 io_poll_tw_hash_eject(req, locked); in io_apoll_task_func()
367 io_req_complete_post(req); in io_apoll_task_func()
369 io_req_task_submit(req, locked); in io_apoll_task_func()
371 io_req_complete_failed(req, ret); in io_apoll_task_func()
374 static void __io_poll_execute(struct io_kiocb *req, int mask) in __io_poll_execute() argument
376 io_req_set_res(req, mask, 0); in __io_poll_execute()
383 if (req->opcode == IORING_OP_POLL_ADD) in __io_poll_execute()
384 req->io_task_work.func = io_poll_task_func; in __io_poll_execute()
386 req->io_task_work.func = io_apoll_task_func; in __io_poll_execute()
388 trace_io_uring_task_add(req, mask); in __io_poll_execute()
389 io_req_task_work_add(req); in __io_poll_execute()
392 static inline void io_poll_execute(struct io_kiocb *req, int res) in io_poll_execute() argument
394 if (io_poll_get_ownership(req)) in io_poll_execute()
395 __io_poll_execute(req, res); in io_poll_execute()
398 static void io_poll_cancel_req(struct io_kiocb *req) in io_poll_cancel_req() argument
400 io_poll_mark_cancelled(req); in io_poll_cancel_req()
402 io_poll_execute(req, 0); in io_poll_cancel_req()
407 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) in io_pollfree_wake() argument
409 io_poll_mark_cancelled(req); in io_pollfree_wake()
411 io_poll_execute(req, 0); in io_pollfree_wake()
435 struct io_kiocb *req = wqe_to_req(wait); in io_poll_wake() local
440 return io_pollfree_wake(req, poll); in io_poll_wake()
446 if (io_poll_get_ownership(req)) { in io_poll_wake()
460 req->flags &= ~REQ_F_DOUBLE_POLL; in io_poll_wake()
462 req->flags &= ~REQ_F_SINGLE_POLL; in io_poll_wake()
464 __io_poll_execute(req, mask); in io_poll_wake()
470 static bool io_poll_double_prepare(struct io_kiocb *req) in io_poll_double_prepare() argument
473 struct io_poll *poll = io_poll_get_single(req); in io_poll_double_prepare()
486 req->flags |= REQ_F_DOUBLE_POLL; in io_poll_double_prepare()
487 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_double_prepare()
488 req->flags |= REQ_F_ASYNC_DATA; in io_poll_double_prepare()
499 struct io_kiocb *req = pt->req; in __io_queue_proc() local
500 unsigned long wqe_private = (unsigned long) req; in __io_queue_proc()
530 if (!io_poll_double_prepare(req)) { in __io_queue_proc()
538 req->flags |= REQ_F_SINGLE_POLL; in __io_queue_proc()
555 struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); in io_poll_queue_proc()
558 (struct io_poll **) &pt->req->async_data); in io_poll_queue_proc()
561 static bool io_poll_can_finish_inline(struct io_kiocb *req, in io_poll_can_finish_inline() argument
564 return pt->owning || io_poll_get_ownership(req); in io_poll_can_finish_inline()
567 static void io_poll_add_hash(struct io_kiocb *req) in io_poll_add_hash() argument
569 if (req->flags & REQ_F_HASH_LOCKED) in io_poll_add_hash()
570 io_poll_req_insert_locked(req); in io_poll_add_hash()
572 io_poll_req_insert(req); in io_poll_add_hash()
581 static int __io_arm_poll_handler(struct io_kiocb *req, in __io_arm_poll_handler() argument
586 struct io_ring_ctx *ctx = req->ctx; in __io_arm_poll_handler()
588 INIT_HLIST_NODE(&req->hash_node); in __io_arm_poll_handler()
589 req->work.cancel_seq = atomic_read(&ctx->cancel_seq); in __io_arm_poll_handler()
591 poll->file = req->file; in __io_arm_poll_handler()
592 req->apoll_events = poll->events; in __io_arm_poll_handler()
595 ipt->req = req; in __io_arm_poll_handler()
610 atomic_set(&req->poll_refs, (int)ipt->owning); in __io_arm_poll_handler()
614 req->flags &= ~REQ_F_HASH_LOCKED; in __io_arm_poll_handler()
616 mask = vfs_poll(req->file, &ipt->pt) & poll->events; in __io_arm_poll_handler()
619 io_poll_remove_entries(req); in __io_arm_poll_handler()
621 if (!io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
622 io_poll_mark_cancelled(req); in __io_arm_poll_handler()
633 if (!io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
634 io_poll_add_hash(req); in __io_arm_poll_handler()
637 io_poll_remove_entries(req); in __io_arm_poll_handler()
643 io_poll_add_hash(req); in __io_arm_poll_handler()
646 io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
647 __io_poll_execute(req, mask); in __io_arm_poll_handler()
656 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) in __io_arm_poll_handler()
657 __io_poll_execute(req, 0); in __io_arm_poll_handler()
666 struct async_poll *apoll = pt->req->apoll; in io_async_queue_proc()
671 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, in io_req_alloc_apoll() argument
674 struct io_ring_ctx *ctx = req->ctx; in io_req_alloc_apoll()
678 if (req->flags & REQ_F_POLLED) { in io_req_alloc_apoll()
679 apoll = req->apoll; in io_req_alloc_apoll()
690 req->apoll = apoll; in io_req_alloc_apoll()
694 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) in io_arm_poll_handler() argument
696 const struct io_op_def *def = &io_op_defs[req->opcode]; in io_arm_poll_handler()
706 req->flags |= REQ_F_HASH_LOCKED; in io_arm_poll_handler()
710 if (!file_can_poll(req->file)) in io_arm_poll_handler()
712 if ((req->flags & (REQ_F_POLLED|REQ_F_PARTIAL_IO)) == REQ_F_POLLED) in io_arm_poll_handler()
714 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) in io_arm_poll_handler()
721 if (req->flags & REQ_F_CLEAR_POLLIN) in io_arm_poll_handler()
729 apoll = io_req_alloc_apoll(req, issue_flags); in io_arm_poll_handler()
732 req->flags |= REQ_F_POLLED; in io_arm_poll_handler()
735 io_kbuf_recycle(req, issue_flags); in io_arm_poll_handler()
737 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); in io_arm_poll_handler()
740 trace_io_uring_poll_arm(req, mask, apoll->poll.events); in io_arm_poll_handler()
750 struct io_kiocb *req; in io_poll_remove_all_table() local
758 hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { in io_poll_remove_all_table()
759 if (io_match_task_safe(req, tsk, cancel_all)) { in io_poll_remove_all_table()
760 hlist_del_init(&req->hash_node); in io_poll_remove_all_table()
761 io_poll_cancel_req(req); in io_poll_remove_all_table()
789 struct io_kiocb *req; in io_poll_find() local
796 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_find()
797 if (cd->data != req->cqe.user_data) in io_poll_find()
799 if (poll_only && req->opcode != IORING_OP_POLL_ADD) in io_poll_find()
802 if (cd->seq == req->work.cancel_seq) in io_poll_find()
804 req->work.cancel_seq = cd->seq; in io_poll_find()
807 return req; in io_poll_find()
819 struct io_kiocb *req; in io_poll_file_find() local
828 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_file_find()
830 req->file != cd->file) in io_poll_file_find()
832 if (cd->seq == req->work.cancel_seq) in io_poll_file_find()
834 req->work.cancel_seq = cd->seq; in io_poll_file_find()
836 return req; in io_poll_file_find()
843 static int io_poll_disarm(struct io_kiocb *req) in io_poll_disarm() argument
845 if (!req) in io_poll_disarm()
847 if (!io_poll_get_ownership(req)) in io_poll_disarm()
849 io_poll_remove_entries(req); in io_poll_disarm()
850 hash_del(&req->hash_node); in io_poll_disarm()
858 struct io_kiocb *req; in __io_poll_cancel() local
861 req = io_poll_file_find(ctx, cd, table, &bucket); in __io_poll_cancel()
863 req = io_poll_find(ctx, false, cd, table, &bucket); in __io_poll_cancel()
865 if (req) in __io_poll_cancel()
866 io_poll_cancel_req(req); in __io_poll_cancel()
869 return req ? 0 : -ENOENT; in __io_poll_cancel()
904 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_poll_remove_prep() argument
906 struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update); in io_poll_remove_prep()
934 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_poll_add_prep() argument
936 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add_prep()
944 if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) in io_poll_add_prep()
951 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) in io_poll_add() argument
953 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add()
963 if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER)) in io_poll_add()
964 req->flags |= REQ_F_HASH_LOCKED; in io_poll_add()
966 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); in io_poll_add()
968 io_req_set_res(req, ipt.result_mask, 0); in io_poll_add()
974 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) in io_poll_remove() argument
976 struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); in io_poll_remove()
978 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove()
1036 req_set_fail(req); in io_poll_remove()
1040 io_req_set_res(req, ret, 0); in io_poll_remove()