1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Generic wait-for-completion handler;
5  *
6  * It differs from semaphores in that their default case is the opposite,
7  * wait_for_completion default blocks whereas semaphore default non-block. The
8  * interface also makes it easy to 'complete' multiple waiting threads,
9  * something which isn't entirely natural for semaphores.
10  *
11  * But more importantly, the primitive documents the usage. Semaphores would
12  * typically be used for exclusion which gives rise to priority inversion.
13  * Waiting for completion is a typically sync point, but not an exclusion point.
14  */
15 
complete_with_flags(struct completion * x,int wake_flags)16 static void complete_with_flags(struct completion *x, int wake_flags)
17 {
18 	unsigned long flags;
19 
20 	raw_spin_lock_irqsave(&x->wait.lock, flags);
21 
22 	if (x->done != UINT_MAX)
23 		x->done++;
24 	swake_up_locked(&x->wait, wake_flags);
25 	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
26 }
27 
complete_on_current_cpu(struct completion * x)28 void complete_on_current_cpu(struct completion *x)
29 {
30 	return complete_with_flags(x, WF_CURRENT_CPU);
31 }
32 
33 /**
34  * complete: - signals a single thread waiting on this completion
35  * @x:  holds the state of this particular completion
36  *
37  * This will wake up a single thread waiting on this completion. Threads will be
38  * awakened in the same order in which they were queued.
39  *
40  * See also complete_all(), wait_for_completion() and related routines.
41  *
42  * If this function wakes up a task, it executes a full memory barrier before
43  * accessing the task state.
44  */
complete(struct completion * x)45 void complete(struct completion *x)
46 {
47 	complete_with_flags(x, 0);
48 }
49 EXPORT_SYMBOL(complete);
50 
51 /**
52  * complete_all: - signals all threads waiting on this completion
53  * @x:  holds the state of this particular completion
54  *
55  * This will wake up all threads waiting on this particular completion event.
56  *
57  * If this function wakes up a task, it executes a full memory barrier before
58  * accessing the task state.
59  *
60  * Since complete_all() sets the completion of @x permanently to done
61  * to allow multiple waiters to finish, a call to reinit_completion()
62  * must be used on @x if @x is to be used again. The code must make
63  * sure that all waiters have woken and finished before reinitializing
64  * @x. Also note that the function completion_done() can not be used
65  * to know if there are still waiters after complete_all() has been called.
66  */
complete_all(struct completion * x)67 void complete_all(struct completion *x)
68 {
69 	unsigned long flags;
70 
71 	lockdep_assert_RT_in_threaded_ctx();
72 
73 	raw_spin_lock_irqsave(&x->wait.lock, flags);
74 	x->done = UINT_MAX;
75 	swake_up_all_locked(&x->wait);
76 	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
77 }
78 EXPORT_SYMBOL(complete_all);
79 
80 static inline long __sched
do_wait_for_common(struct completion * x,long (* action)(long),long timeout,int state)81 do_wait_for_common(struct completion *x,
82 		   long (*action)(long), long timeout, int state)
83 {
84 	if (!x->done) {
85 		DECLARE_SWAITQUEUE(wait);
86 
87 		do {
88 			if (signal_pending_state(state, current)) {
89 				timeout = -ERESTARTSYS;
90 				break;
91 			}
92 			__prepare_to_swait(&x->wait, &wait);
93 			__set_current_state(state);
94 			raw_spin_unlock_irq(&x->wait.lock);
95 			timeout = action(timeout);
96 			raw_spin_lock_irq(&x->wait.lock);
97 		} while (!x->done && timeout);
98 		__finish_swait(&x->wait, &wait);
99 		if (!x->done)
100 			return timeout;
101 	}
102 	if (x->done != UINT_MAX)
103 		x->done--;
104 	return timeout ?: 1;
105 }
106 
107 static inline long __sched
__wait_for_common(struct completion * x,long (* action)(long),long timeout,int state)108 __wait_for_common(struct completion *x,
109 		  long (*action)(long), long timeout, int state)
110 {
111 	might_sleep();
112 
113 	complete_acquire(x);
114 
115 	raw_spin_lock_irq(&x->wait.lock);
116 	timeout = do_wait_for_common(x, action, timeout, state);
117 	raw_spin_unlock_irq(&x->wait.lock);
118 
119 	complete_release(x);
120 
121 	return timeout;
122 }
123 
124 static long __sched
wait_for_common(struct completion * x,long timeout,int state)125 wait_for_common(struct completion *x, long timeout, int state)
126 {
127 	return __wait_for_common(x, schedule_timeout, timeout, state);
128 }
129 
130 static long __sched
wait_for_common_io(struct completion * x,long timeout,int state)131 wait_for_common_io(struct completion *x, long timeout, int state)
132 {
133 	return __wait_for_common(x, io_schedule_timeout, timeout, state);
134 }
135 
136 /**
137  * wait_for_completion: - waits for completion of a task
138  * @x:  holds the state of this particular completion
139  *
140  * This waits to be signaled for completion of a specific task. It is NOT
141  * interruptible and there is no timeout.
142  *
143  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
144  * and interrupt capability. Also see complete().
145  */
wait_for_completion(struct completion * x)146 void __sched wait_for_completion(struct completion *x)
147 {
148 	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
149 }
150 EXPORT_SYMBOL(wait_for_completion);
151 
152 /**
153  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
154  * @x:  holds the state of this particular completion
155  * @timeout:  timeout value in jiffies
156  *
157  * This waits for either a completion of a specific task to be signaled or for a
158  * specified timeout to expire. The timeout is in jiffies. It is not
159  * interruptible.
160  *
161  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
162  * till timeout) if completed.
163  */
164 unsigned long __sched
wait_for_completion_timeout(struct completion * x,unsigned long timeout)165 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
166 {
167 	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
168 }
169 EXPORT_SYMBOL(wait_for_completion_timeout);
170 
171 /**
172  * wait_for_completion_io: - waits for completion of a task
173  * @x:  holds the state of this particular completion
174  *
175  * This waits to be signaled for completion of a specific task. It is NOT
176  * interruptible and there is no timeout. The caller is accounted as waiting
177  * for IO (which traditionally means blkio only).
178  */
wait_for_completion_io(struct completion * x)179 void __sched wait_for_completion_io(struct completion *x)
180 {
181 	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
182 }
183 EXPORT_SYMBOL(wait_for_completion_io);
184 
185 /**
186  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
187  * @x:  holds the state of this particular completion
188  * @timeout:  timeout value in jiffies
189  *
190  * This waits for either a completion of a specific task to be signaled or for a
191  * specified timeout to expire. The timeout is in jiffies. It is not
192  * interruptible. The caller is accounted as waiting for IO (which traditionally
193  * means blkio only).
194  *
195  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
196  * till timeout) if completed.
197  */
198 unsigned long __sched
wait_for_completion_io_timeout(struct completion * x,unsigned long timeout)199 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
200 {
201 	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
202 }
203 EXPORT_SYMBOL(wait_for_completion_io_timeout);
204 
205 /**
206  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
207  * @x:  holds the state of this particular completion
208  *
209  * This waits for completion of a specific task to be signaled. It is
210  * interruptible.
211  *
212  * Return: -ERESTARTSYS if interrupted, 0 if completed.
213  */
wait_for_completion_interruptible(struct completion * x)214 int __sched wait_for_completion_interruptible(struct completion *x)
215 {
216 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
217 
218 	if (t == -ERESTARTSYS)
219 		return t;
220 	return 0;
221 }
222 EXPORT_SYMBOL(wait_for_completion_interruptible);
223 
224 /**
225  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
226  * @x:  holds the state of this particular completion
227  * @timeout:  timeout value in jiffies
228  *
229  * This waits for either a completion of a specific task to be signaled or for a
230  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
231  *
232  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
233  * or number of jiffies left till timeout) if completed.
234  */
235 long __sched
wait_for_completion_interruptible_timeout(struct completion * x,unsigned long timeout)236 wait_for_completion_interruptible_timeout(struct completion *x,
237 					  unsigned long timeout)
238 {
239 	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
240 }
241 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
242 
243 /**
244  * wait_for_completion_killable: - waits for completion of a task (killable)
245  * @x:  holds the state of this particular completion
246  *
247  * This waits to be signaled for completion of a specific task. It can be
248  * interrupted by a kill signal.
249  *
250  * Return: -ERESTARTSYS if interrupted, 0 if completed.
251  */
wait_for_completion_killable(struct completion * x)252 int __sched wait_for_completion_killable(struct completion *x)
253 {
254 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
255 
256 	if (t == -ERESTARTSYS)
257 		return t;
258 	return 0;
259 }
260 EXPORT_SYMBOL(wait_for_completion_killable);
261 
wait_for_completion_state(struct completion * x,unsigned int state)262 int __sched wait_for_completion_state(struct completion *x, unsigned int state)
263 {
264 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
265 
266 	if (t == -ERESTARTSYS)
267 		return t;
268 	return 0;
269 }
270 EXPORT_SYMBOL(wait_for_completion_state);
271 
272 /**
273  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
274  * @x:  holds the state of this particular completion
275  * @timeout:  timeout value in jiffies
276  *
277  * This waits for either a completion of a specific task to be
278  * signaled or for a specified timeout to expire. It can be
279  * interrupted by a kill signal. The timeout is in jiffies.
280  *
281  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
282  * or number of jiffies left till timeout) if completed.
283  */
284 long __sched
wait_for_completion_killable_timeout(struct completion * x,unsigned long timeout)285 wait_for_completion_killable_timeout(struct completion *x,
286 				     unsigned long timeout)
287 {
288 	return wait_for_common(x, timeout, TASK_KILLABLE);
289 }
290 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
291 
292 /**
293  *	try_wait_for_completion - try to decrement a completion without blocking
294  *	@x:	completion structure
295  *
296  *	Return: 0 if a decrement cannot be done without blocking
297  *		 1 if a decrement succeeded.
298  *
299  *	If a completion is being used as a counting completion,
300  *	attempt to decrement the counter without blocking. This
301  *	enables us to avoid waiting if the resource the completion
302  *	is protecting is not available.
303  */
try_wait_for_completion(struct completion * x)304 bool try_wait_for_completion(struct completion *x)
305 {
306 	unsigned long flags;
307 	bool ret = true;
308 
309 	/*
310 	 * Since x->done will need to be locked only
311 	 * in the non-blocking case, we check x->done
312 	 * first without taking the lock so we can
313 	 * return early in the blocking case.
314 	 */
315 	if (!READ_ONCE(x->done))
316 		return false;
317 
318 	raw_spin_lock_irqsave(&x->wait.lock, flags);
319 	if (!x->done)
320 		ret = false;
321 	else if (x->done != UINT_MAX)
322 		x->done--;
323 	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
324 	return ret;
325 }
326 EXPORT_SYMBOL(try_wait_for_completion);
327 
328 /**
329  *	completion_done - Test to see if a completion has any waiters
330  *	@x:	completion structure
331  *
332  *	Return: 0 if there are waiters (wait_for_completion() in progress)
333  *		 1 if there are no waiters.
334  *
335  *	Note, this will always return true if complete_all() was called on @X.
336  */
completion_done(struct completion * x)337 bool completion_done(struct completion *x)
338 {
339 	unsigned long flags;
340 
341 	if (!READ_ONCE(x->done))
342 		return false;
343 
344 	/*
345 	 * If ->done, we need to wait for complete() to release ->wait.lock
346 	 * otherwise we can end up freeing the completion before complete()
347 	 * is done referencing it.
348 	 */
349 	raw_spin_lock_irqsave(&x->wait.lock, flags);
350 	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
351 	return true;
352 }
353 EXPORT_SYMBOL(completion_done);
354