1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * A generic kernel FIFO implementation
4  *
5  * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
6  */
7 
8 #ifndef _LINUX_KFIFO_H
9 #define _LINUX_KFIFO_H
10 
11 /*
12  * How to porting drivers to the new generic FIFO API:
13  *
14  * - Modify the declaration of the "struct kfifo *" object into a
15  *   in-place "struct kfifo" object
16  * - Init the in-place object with kfifo_alloc() or kfifo_init()
17  *   Note: The address of the in-place "struct kfifo" object must be
18  *   passed as the first argument to this functions
19  * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
20  *   into kfifo_out
21  * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
22  *   into kfifo_out_spinlocked
23  *   Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
24  *   must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
25  *   as the last parameter
26  * - The formerly __kfifo_* functions are renamed into kfifo_*
27  */
28 
29 /*
30  * Note about locking: There is no locking required until only one reader
31  * and one writer is using the fifo and no kfifo_reset() will be called.
32  * kfifo_reset_out() can be safely used, until it will be only called
33  * in the reader thread.
34  * For multiple writer and one reader there is only a need to lock the writer.
35  * And vice versa for only one writer and multiple reader there is only a need
36  * to lock the reader.
37  */
38 
39 #include <linux/kernel.h>
40 #include <linux/spinlock.h>
41 #include <linux/stddef.h>
42 #include <linux/scatterlist.h>
43 
44 struct __kfifo {
45 	unsigned int	in;
46 	unsigned int	out;
47 	unsigned int	mask;
48 	unsigned int	esize;
49 	void		*data;
50 };
51 
52 #define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
53 	union { \
54 		struct __kfifo	kfifo; \
55 		datatype	*type; \
56 		const datatype	*const_type; \
57 		char		(*rectype)[recsize]; \
58 		ptrtype		*ptr; \
59 		ptrtype const	*ptr_const; \
60 	}
61 
62 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
63 { \
64 	__STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
65 	type		buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
66 }
67 
68 #define STRUCT_KFIFO(type, size) \
69 	struct __STRUCT_KFIFO(type, size, 0, type)
70 
71 #define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
72 { \
73 	__STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
74 	type		buf[0]; \
75 }
76 
77 #define STRUCT_KFIFO_PTR(type) \
78 	struct __STRUCT_KFIFO_PTR(type, 0, type)
79 
80 /*
81  * define compatibility "struct kfifo" for dynamic allocated fifos
82  */
83 struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
84 
85 #define STRUCT_KFIFO_REC_1(size) \
86 	struct __STRUCT_KFIFO(unsigned char, size, 1, void)
87 
88 #define STRUCT_KFIFO_REC_2(size) \
89 	struct __STRUCT_KFIFO(unsigned char, size, 2, void)
90 
91 /*
92  * define kfifo_rec types
93  */
94 struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
95 struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
96 
97 /*
98  * helper macro to distinguish between real in place fifo where the fifo
99  * array is a part of the structure and the fifo type where the array is
100  * outside of the fifo structure.
101  */
102 #define	__is_kfifo_ptr(fifo) \
103 	(sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
104 
105 /**
106  * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
107  * @fifo: name of the declared fifo
108  * @type: type of the fifo elements
109  */
110 #define DECLARE_KFIFO_PTR(fifo, type)	STRUCT_KFIFO_PTR(type) fifo
111 
112 /**
113  * DECLARE_KFIFO - macro to declare a fifo object
114  * @fifo: name of the declared fifo
115  * @type: type of the fifo elements
116  * @size: the number of elements in the fifo, this must be a power of 2
117  */
118 #define DECLARE_KFIFO(fifo, type, size)	STRUCT_KFIFO(type, size) fifo
119 
120 /**
121  * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
122  * @fifo: name of the declared fifo datatype
123  */
124 #define INIT_KFIFO(fifo) \
125 (void)({ \
126 	typeof(&(fifo)) __tmp = &(fifo); \
127 	struct __kfifo *__kfifo = &__tmp->kfifo; \
128 	__kfifo->in = 0; \
129 	__kfifo->out = 0; \
130 	__kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
131 	__kfifo->esize = sizeof(*__tmp->buf); \
132 	__kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
133 })
134 
135 /**
136  * DEFINE_KFIFO - macro to define and initialize a fifo
137  * @fifo: name of the declared fifo datatype
138  * @type: type of the fifo elements
139  * @size: the number of elements in the fifo, this must be a power of 2
140  *
141  * Note: the macro can be used for global and local fifo data type variables.
142  */
143 #define DEFINE_KFIFO(fifo, type, size) \
144 	DECLARE_KFIFO(fifo, type, size) = \
145 	(typeof(fifo)) { \
146 		{ \
147 			{ \
148 			.in	= 0, \
149 			.out	= 0, \
150 			.mask	= __is_kfifo_ptr(&(fifo)) ? \
151 				  0 : \
152 				  ARRAY_SIZE((fifo).buf) - 1, \
153 			.esize	= sizeof(*(fifo).buf), \
154 			.data	= __is_kfifo_ptr(&(fifo)) ? \
155 				NULL : \
156 				(fifo).buf, \
157 			} \
158 		} \
159 	}
160 
161 
162 static inline unsigned int __must_check
__kfifo_uint_must_check_helper(unsigned int val)163 __kfifo_uint_must_check_helper(unsigned int val)
164 {
165 	return val;
166 }
167 
168 static inline int __must_check
__kfifo_int_must_check_helper(int val)169 __kfifo_int_must_check_helper(int val)
170 {
171 	return val;
172 }
173 
174 /**
175  * kfifo_initialized - Check if the fifo is initialized
176  * @fifo: address of the fifo to check
177  *
178  * Return %true if fifo is initialized, otherwise %false.
179  * Assumes the fifo was 0 before.
180  */
181 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
182 
183 /**
184  * kfifo_esize - returns the size of the element managed by the fifo
185  * @fifo: address of the fifo to be used
186  */
187 #define kfifo_esize(fifo)	((fifo)->kfifo.esize)
188 
189 /**
190  * kfifo_recsize - returns the size of the record length field
191  * @fifo: address of the fifo to be used
192  */
193 #define kfifo_recsize(fifo)	(sizeof(*(fifo)->rectype))
194 
195 /**
196  * kfifo_size - returns the size of the fifo in elements
197  * @fifo: address of the fifo to be used
198  */
199 #define kfifo_size(fifo)	((fifo)->kfifo.mask + 1)
200 
201 /**
202  * kfifo_reset - removes the entire fifo content
203  * @fifo: address of the fifo to be used
204  *
205  * Note: usage of kfifo_reset() is dangerous. It should be only called when the
206  * fifo is exclusived locked or when it is secured that no other thread is
207  * accessing the fifo.
208  */
209 #define kfifo_reset(fifo) \
210 (void)({ \
211 	typeof((fifo) + 1) __tmp = (fifo); \
212 	__tmp->kfifo.in = __tmp->kfifo.out = 0; \
213 })
214 
215 /**
216  * kfifo_reset_out - skip fifo content
217  * @fifo: address of the fifo to be used
218  *
219  * Note: The usage of kfifo_reset_out() is safe until it will be only called
220  * from the reader thread and there is only one concurrent reader. Otherwise
221  * it is dangerous and must be handled in the same way as kfifo_reset().
222  */
223 #define kfifo_reset_out(fifo)	\
224 (void)({ \
225 	typeof((fifo) + 1) __tmp = (fifo); \
226 	__tmp->kfifo.out = __tmp->kfifo.in; \
227 })
228 
229 /**
230  * kfifo_len - returns the number of used elements in the fifo
231  * @fifo: address of the fifo to be used
232  */
233 #define kfifo_len(fifo) \
234 ({ \
235 	typeof((fifo) + 1) __tmpl = (fifo); \
236 	__tmpl->kfifo.in - __tmpl->kfifo.out; \
237 })
238 
239 /**
240  * kfifo_is_empty - returns true if the fifo is empty
241  * @fifo: address of the fifo to be used
242  */
243 #define	kfifo_is_empty(fifo) \
244 ({ \
245 	typeof((fifo) + 1) __tmpq = (fifo); \
246 	__tmpq->kfifo.in == __tmpq->kfifo.out; \
247 })
248 
249 /**
250  * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
251  * a spinlock for locking
252  * @fifo: address of the fifo to be used
253  * @lock: spinlock to be used for locking
254  */
255 #define kfifo_is_empty_spinlocked(fifo, lock) \
256 ({ \
257 	unsigned long __flags; \
258 	bool __ret; \
259 	spin_lock_irqsave(lock, __flags); \
260 	__ret = kfifo_is_empty(fifo); \
261 	spin_unlock_irqrestore(lock, __flags); \
262 	__ret; \
263 })
264 
265 /**
266  * kfifo_is_empty_spinlocked_noirqsave  - returns true if the fifo is empty
267  * using a spinlock for locking, doesn't disable interrupts
268  * @fifo: address of the fifo to be used
269  * @lock: spinlock to be used for locking
270  */
271 #define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \
272 ({ \
273 	bool __ret; \
274 	spin_lock(lock); \
275 	__ret = kfifo_is_empty(fifo); \
276 	spin_unlock(lock); \
277 	__ret; \
278 })
279 
280 /**
281  * kfifo_is_full - returns true if the fifo is full
282  * @fifo: address of the fifo to be used
283  */
284 #define	kfifo_is_full(fifo) \
285 ({ \
286 	typeof((fifo) + 1) __tmpq = (fifo); \
287 	kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
288 })
289 
290 /**
291  * kfifo_avail - returns the number of unused elements in the fifo
292  * @fifo: address of the fifo to be used
293  */
294 #define	kfifo_avail(fifo) \
295 __kfifo_uint_must_check_helper( \
296 ({ \
297 	typeof((fifo) + 1) __tmpq = (fifo); \
298 	const size_t __recsize = sizeof(*__tmpq->rectype); \
299 	unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
300 	(__recsize) ? ((__avail <= __recsize) ? 0 : \
301 	__kfifo_max_r(__avail - __recsize, __recsize)) : \
302 	__avail; \
303 }) \
304 )
305 
306 /**
307  * kfifo_skip - skip output data
308  * @fifo: address of the fifo to be used
309  */
310 #define	kfifo_skip(fifo) \
311 (void)({ \
312 	typeof((fifo) + 1) __tmp = (fifo); \
313 	const size_t __recsize = sizeof(*__tmp->rectype); \
314 	struct __kfifo *__kfifo = &__tmp->kfifo; \
315 	if (__recsize) \
316 		__kfifo_skip_r(__kfifo, __recsize); \
317 	else \
318 		__kfifo->out++; \
319 })
320 
321 /**
322  * kfifo_peek_len - gets the size of the next fifo record
323  * @fifo: address of the fifo to be used
324  *
325  * This function returns the size of the next fifo record in number of bytes.
326  */
327 #define kfifo_peek_len(fifo) \
328 __kfifo_uint_must_check_helper( \
329 ({ \
330 	typeof((fifo) + 1) __tmp = (fifo); \
331 	const size_t __recsize = sizeof(*__tmp->rectype); \
332 	struct __kfifo *__kfifo = &__tmp->kfifo; \
333 	(!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
334 	__kfifo_len_r(__kfifo, __recsize); \
335 }) \
336 )
337 
338 /**
339  * kfifo_alloc - dynamically allocates a new fifo buffer
340  * @fifo: pointer to the fifo
341  * @size: the number of elements in the fifo, this must be a power of 2
342  * @gfp_mask: get_free_pages mask, passed to kmalloc()
343  *
344  * This macro dynamically allocates a new fifo buffer.
345  *
346  * The number of elements will be rounded-up to a power of 2.
347  * The fifo will be release with kfifo_free().
348  * Return 0 if no error, otherwise an error code.
349  */
350 #define kfifo_alloc(fifo, size, gfp_mask) \
351 __kfifo_int_must_check_helper( \
352 ({ \
353 	typeof((fifo) + 1) __tmp = (fifo); \
354 	struct __kfifo *__kfifo = &__tmp->kfifo; \
355 	__is_kfifo_ptr(__tmp) ? \
356 	__kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
357 	-EINVAL; \
358 }) \
359 )
360 
361 /**
362  * kfifo_free - frees the fifo
363  * @fifo: the fifo to be freed
364  */
365 #define kfifo_free(fifo) \
366 ({ \
367 	typeof((fifo) + 1) __tmp = (fifo); \
368 	struct __kfifo *__kfifo = &__tmp->kfifo; \
369 	if (__is_kfifo_ptr(__tmp)) \
370 		__kfifo_free(__kfifo); \
371 })
372 
373 /**
374  * kfifo_init - initialize a fifo using a preallocated buffer
375  * @fifo: the fifo to assign the buffer
376  * @buffer: the preallocated buffer to be used
377  * @size: the size of the internal buffer, this have to be a power of 2
378  *
379  * This macro initializes a fifo using a preallocated buffer.
380  *
381  * The number of elements will be rounded-up to a power of 2.
382  * Return 0 if no error, otherwise an error code.
383  */
384 #define kfifo_init(fifo, buffer, size) \
385 ({ \
386 	typeof((fifo) + 1) __tmp = (fifo); \
387 	struct __kfifo *__kfifo = &__tmp->kfifo; \
388 	__is_kfifo_ptr(__tmp) ? \
389 	__kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
390 	-EINVAL; \
391 })
392 
393 /**
394  * kfifo_put - put data into the fifo
395  * @fifo: address of the fifo to be used
396  * @val: the data to be added
397  *
398  * This macro copies the given value into the fifo.
399  * It returns 0 if the fifo was full. Otherwise it returns the number
400  * processed elements.
401  *
402  * Note that with only one concurrent reader and one concurrent
403  * writer, you don't need extra locking to use these macro.
404  */
405 #define	kfifo_put(fifo, val) \
406 ({ \
407 	typeof((fifo) + 1) __tmp = (fifo); \
408 	typeof(*__tmp->const_type) __val = (val); \
409 	unsigned int __ret; \
410 	size_t __recsize = sizeof(*__tmp->rectype); \
411 	struct __kfifo *__kfifo = &__tmp->kfifo; \
412 	if (__recsize) \
413 		__ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
414 			__recsize); \
415 	else { \
416 		__ret = !kfifo_is_full(__tmp); \
417 		if (__ret) { \
418 			(__is_kfifo_ptr(__tmp) ? \
419 			((typeof(__tmp->type))__kfifo->data) : \
420 			(__tmp->buf) \
421 			)[__kfifo->in & __tmp->kfifo.mask] = \
422 				*(typeof(__tmp->type))&__val; \
423 			smp_wmb(); \
424 			__kfifo->in++; \
425 		} \
426 	} \
427 	__ret; \
428 })
429 
430 /**
431  * kfifo_get - get data from the fifo
432  * @fifo: address of the fifo to be used
433  * @val: address where to store the data
434  *
435  * This macro reads the data from the fifo.
436  * It returns 0 if the fifo was empty. Otherwise it returns the number
437  * processed elements.
438  *
439  * Note that with only one concurrent reader and one concurrent
440  * writer, you don't need extra locking to use these macro.
441  */
442 #define	kfifo_get(fifo, val) \
443 __kfifo_uint_must_check_helper( \
444 ({ \
445 	typeof((fifo) + 1) __tmp = (fifo); \
446 	typeof(__tmp->ptr) __val = (val); \
447 	unsigned int __ret; \
448 	const size_t __recsize = sizeof(*__tmp->rectype); \
449 	struct __kfifo *__kfifo = &__tmp->kfifo; \
450 	if (__recsize) \
451 		__ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
452 			__recsize); \
453 	else { \
454 		__ret = !kfifo_is_empty(__tmp); \
455 		if (__ret) { \
456 			*(typeof(__tmp->type))__val = \
457 				(__is_kfifo_ptr(__tmp) ? \
458 				((typeof(__tmp->type))__kfifo->data) : \
459 				(__tmp->buf) \
460 				)[__kfifo->out & __tmp->kfifo.mask]; \
461 			smp_wmb(); \
462 			__kfifo->out++; \
463 		} \
464 	} \
465 	__ret; \
466 }) \
467 )
468 
469 /**
470  * kfifo_peek - get data from the fifo without removing
471  * @fifo: address of the fifo to be used
472  * @val: address where to store the data
473  *
474  * This reads the data from the fifo without removing it from the fifo.
475  * It returns 0 if the fifo was empty. Otherwise it returns the number
476  * processed elements.
477  *
478  * Note that with only one concurrent reader and one concurrent
479  * writer, you don't need extra locking to use these macro.
480  */
481 #define	kfifo_peek(fifo, val) \
482 __kfifo_uint_must_check_helper( \
483 ({ \
484 	typeof((fifo) + 1) __tmp = (fifo); \
485 	typeof(__tmp->ptr) __val = (val); \
486 	unsigned int __ret; \
487 	const size_t __recsize = sizeof(*__tmp->rectype); \
488 	struct __kfifo *__kfifo = &__tmp->kfifo; \
489 	if (__recsize) \
490 		__ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
491 			__recsize); \
492 	else { \
493 		__ret = !kfifo_is_empty(__tmp); \
494 		if (__ret) { \
495 			*(typeof(__tmp->type))__val = \
496 				(__is_kfifo_ptr(__tmp) ? \
497 				((typeof(__tmp->type))__kfifo->data) : \
498 				(__tmp->buf) \
499 				)[__kfifo->out & __tmp->kfifo.mask]; \
500 			smp_wmb(); \
501 		} \
502 	} \
503 	__ret; \
504 }) \
505 )
506 
507 /**
508  * kfifo_in - put data into the fifo
509  * @fifo: address of the fifo to be used
510  * @buf: the data to be added
511  * @n: number of elements to be added
512  *
513  * This macro copies the given buffer into the fifo and returns the
514  * number of copied elements.
515  *
516  * Note that with only one concurrent reader and one concurrent
517  * writer, you don't need extra locking to use these macro.
518  */
519 #define	kfifo_in(fifo, buf, n) \
520 ({ \
521 	typeof((fifo) + 1) __tmp = (fifo); \
522 	typeof(__tmp->ptr_const) __buf = (buf); \
523 	unsigned long __n = (n); \
524 	const size_t __recsize = sizeof(*__tmp->rectype); \
525 	struct __kfifo *__kfifo = &__tmp->kfifo; \
526 	(__recsize) ?\
527 	__kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
528 	__kfifo_in(__kfifo, __buf, __n); \
529 })
530 
531 /**
532  * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
533  * @fifo: address of the fifo to be used
534  * @buf: the data to be added
535  * @n: number of elements to be added
536  * @lock: pointer to the spinlock to use for locking
537  *
538  * This macro copies the given values buffer into the fifo and returns the
539  * number of copied elements.
540  */
541 #define	kfifo_in_spinlocked(fifo, buf, n, lock) \
542 ({ \
543 	unsigned long __flags; \
544 	unsigned int __ret; \
545 	spin_lock_irqsave(lock, __flags); \
546 	__ret = kfifo_in(fifo, buf, n); \
547 	spin_unlock_irqrestore(lock, __flags); \
548 	__ret; \
549 })
550 
551 /**
552  * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
553  * locking, don't disable interrupts
554  * @fifo: address of the fifo to be used
555  * @buf: the data to be added
556  * @n: number of elements to be added
557  * @lock: pointer to the spinlock to use for locking
558  *
559  * This is a variant of kfifo_in_spinlocked() but uses spin_lock/unlock()
560  * for locking and doesn't disable interrupts.
561  */
562 #define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \
563 ({ \
564 	unsigned int __ret; \
565 	spin_lock(lock); \
566 	__ret = kfifo_in(fifo, buf, n); \
567 	spin_unlock(lock); \
568 	__ret; \
569 })
570 
571 /* alias for kfifo_in_spinlocked, will be removed in a future release */
572 #define kfifo_in_locked(fifo, buf, n, lock) \
573 		kfifo_in_spinlocked(fifo, buf, n, lock)
574 
575 /**
576  * kfifo_out - get data from the fifo
577  * @fifo: address of the fifo to be used
578  * @buf: pointer to the storage buffer
579  * @n: max. number of elements to get
580  *
581  * This macro get some data from the fifo and return the numbers of elements
582  * copied.
583  *
584  * Note that with only one concurrent reader and one concurrent
585  * writer, you don't need extra locking to use these macro.
586  */
587 #define	kfifo_out(fifo, buf, n) \
588 __kfifo_uint_must_check_helper( \
589 ({ \
590 	typeof((fifo) + 1) __tmp = (fifo); \
591 	typeof(__tmp->ptr) __buf = (buf); \
592 	unsigned long __n = (n); \
593 	const size_t __recsize = sizeof(*__tmp->rectype); \
594 	struct __kfifo *__kfifo = &__tmp->kfifo; \
595 	(__recsize) ?\
596 	__kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
597 	__kfifo_out(__kfifo, __buf, __n); \
598 }) \
599 )
600 
601 /**
602  * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
603  * @fifo: address of the fifo to be used
604  * @buf: pointer to the storage buffer
605  * @n: max. number of elements to get
606  * @lock: pointer to the spinlock to use for locking
607  *
608  * This macro get the data from the fifo and return the numbers of elements
609  * copied.
610  */
611 #define	kfifo_out_spinlocked(fifo, buf, n, lock) \
612 __kfifo_uint_must_check_helper( \
613 ({ \
614 	unsigned long __flags; \
615 	unsigned int __ret; \
616 	spin_lock_irqsave(lock, __flags); \
617 	__ret = kfifo_out(fifo, buf, n); \
618 	spin_unlock_irqrestore(lock, __flags); \
619 	__ret; \
620 }) \
621 )
622 
623 /**
624  * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
625  * for locking, don't disable interrupts
626  * @fifo: address of the fifo to be used
627  * @buf: pointer to the storage buffer
628  * @n: max. number of elements to get
629  * @lock: pointer to the spinlock to use for locking
630  *
631  * This is a variant of kfifo_out_spinlocked() which uses spin_lock/unlock()
632  * for locking and doesn't disable interrupts.
633  */
634 #define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \
635 __kfifo_uint_must_check_helper( \
636 ({ \
637 	unsigned int __ret; \
638 	spin_lock(lock); \
639 	__ret = kfifo_out(fifo, buf, n); \
640 	spin_unlock(lock); \
641 	__ret; \
642 }) \
643 )
644 
645 /* alias for kfifo_out_spinlocked, will be removed in a future release */
646 #define kfifo_out_locked(fifo, buf, n, lock) \
647 		kfifo_out_spinlocked(fifo, buf, n, lock)
648 
649 /**
650  * kfifo_from_user - puts some data from user space into the fifo
651  * @fifo: address of the fifo to be used
652  * @from: pointer to the data to be added
653  * @len: the length of the data to be added
654  * @copied: pointer to output variable to store the number of copied bytes
655  *
656  * This macro copies at most @len bytes from the @from into the
657  * fifo, depending of the available space and returns -EFAULT/0.
658  *
659  * Note that with only one concurrent reader and one concurrent
660  * writer, you don't need extra locking to use these macro.
661  */
662 #define	kfifo_from_user(fifo, from, len, copied) \
663 __kfifo_uint_must_check_helper( \
664 ({ \
665 	typeof((fifo) + 1) __tmp = (fifo); \
666 	const void __user *__from = (from); \
667 	unsigned int __len = (len); \
668 	unsigned int *__copied = (copied); \
669 	const size_t __recsize = sizeof(*__tmp->rectype); \
670 	struct __kfifo *__kfifo = &__tmp->kfifo; \
671 	(__recsize) ? \
672 	__kfifo_from_user_r(__kfifo, __from, __len,  __copied, __recsize) : \
673 	__kfifo_from_user(__kfifo, __from, __len, __copied); \
674 }) \
675 )
676 
677 /**
678  * kfifo_to_user - copies data from the fifo into user space
679  * @fifo: address of the fifo to be used
680  * @to: where the data must be copied
681  * @len: the size of the destination buffer
682  * @copied: pointer to output variable to store the number of copied bytes
683  *
684  * This macro copies at most @len bytes from the fifo into the
685  * @to buffer and returns -EFAULT/0.
686  *
687  * Note that with only one concurrent reader and one concurrent
688  * writer, you don't need extra locking to use these macro.
689  */
690 #define	kfifo_to_user(fifo, to, len, copied) \
691 __kfifo_int_must_check_helper( \
692 ({ \
693 	typeof((fifo) + 1) __tmp = (fifo); \
694 	void __user *__to = (to); \
695 	unsigned int __len = (len); \
696 	unsigned int *__copied = (copied); \
697 	const size_t __recsize = sizeof(*__tmp->rectype); \
698 	struct __kfifo *__kfifo = &__tmp->kfifo; \
699 	(__recsize) ? \
700 	__kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
701 	__kfifo_to_user(__kfifo, __to, __len, __copied); \
702 }) \
703 )
704 
705 /**
706  * kfifo_dma_in_prepare - setup a scatterlist for DMA input
707  * @fifo: address of the fifo to be used
708  * @sgl: pointer to the scatterlist array
709  * @nents: number of entries in the scatterlist array
710  * @len: number of elements to transfer
711  *
712  * This macro fills a scatterlist for DMA input.
713  * It returns the number entries in the scatterlist array.
714  *
715  * Note that with only one concurrent reader and one concurrent
716  * writer, you don't need extra locking to use these macros.
717  */
718 #define	kfifo_dma_in_prepare(fifo, sgl, nents, len) \
719 ({ \
720 	typeof((fifo) + 1) __tmp = (fifo); \
721 	struct scatterlist *__sgl = (sgl); \
722 	int __nents = (nents); \
723 	unsigned int __len = (len); \
724 	const size_t __recsize = sizeof(*__tmp->rectype); \
725 	struct __kfifo *__kfifo = &__tmp->kfifo; \
726 	(__recsize) ? \
727 	__kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
728 	__kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
729 })
730 
731 /**
732  * kfifo_dma_in_finish - finish a DMA IN operation
733  * @fifo: address of the fifo to be used
734  * @len: number of bytes to received
735  *
736  * This macro finish a DMA IN operation. The in counter will be updated by
737  * the len parameter. No error checking will be done.
738  *
739  * Note that with only one concurrent reader and one concurrent
740  * writer, you don't need extra locking to use these macros.
741  */
742 #define kfifo_dma_in_finish(fifo, len) \
743 (void)({ \
744 	typeof((fifo) + 1) __tmp = (fifo); \
745 	unsigned int __len = (len); \
746 	const size_t __recsize = sizeof(*__tmp->rectype); \
747 	struct __kfifo *__kfifo = &__tmp->kfifo; \
748 	if (__recsize) \
749 		__kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
750 	else \
751 		__kfifo->in += __len / sizeof(*__tmp->type); \
752 })
753 
754 /**
755  * kfifo_dma_out_prepare - setup a scatterlist for DMA output
756  * @fifo: address of the fifo to be used
757  * @sgl: pointer to the scatterlist array
758  * @nents: number of entries in the scatterlist array
759  * @len: number of elements to transfer
760  *
761  * This macro fills a scatterlist for DMA output which at most @len bytes
762  * to transfer.
763  * It returns the number entries in the scatterlist array.
764  * A zero means there is no space available and the scatterlist is not filled.
765  *
766  * Note that with only one concurrent reader and one concurrent
767  * writer, you don't need extra locking to use these macros.
768  */
769 #define	kfifo_dma_out_prepare(fifo, sgl, nents, len) \
770 ({ \
771 	typeof((fifo) + 1) __tmp = (fifo);  \
772 	struct scatterlist *__sgl = (sgl); \
773 	int __nents = (nents); \
774 	unsigned int __len = (len); \
775 	const size_t __recsize = sizeof(*__tmp->rectype); \
776 	struct __kfifo *__kfifo = &__tmp->kfifo; \
777 	(__recsize) ? \
778 	__kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
779 	__kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
780 })
781 
782 /**
783  * kfifo_dma_out_finish - finish a DMA OUT operation
784  * @fifo: address of the fifo to be used
785  * @len: number of bytes transferred
786  *
787  * This macro finish a DMA OUT operation. The out counter will be updated by
788  * the len parameter. No error checking will be done.
789  *
790  * Note that with only one concurrent reader and one concurrent
791  * writer, you don't need extra locking to use these macros.
792  */
793 #define kfifo_dma_out_finish(fifo, len) \
794 (void)({ \
795 	typeof((fifo) + 1) __tmp = (fifo); \
796 	unsigned int __len = (len); \
797 	const size_t __recsize = sizeof(*__tmp->rectype); \
798 	struct __kfifo *__kfifo = &__tmp->kfifo; \
799 	if (__recsize) \
800 		__kfifo_dma_out_finish_r(__kfifo, __recsize); \
801 	else \
802 		__kfifo->out += __len / sizeof(*__tmp->type); \
803 })
804 
805 /**
806  * kfifo_out_peek - gets some data from the fifo
807  * @fifo: address of the fifo to be used
808  * @buf: pointer to the storage buffer
809  * @n: max. number of elements to get
810  *
811  * This macro get the data from the fifo and return the numbers of elements
812  * copied. The data is not removed from the fifo.
813  *
814  * Note that with only one concurrent reader and one concurrent
815  * writer, you don't need extra locking to use these macro.
816  */
817 #define	kfifo_out_peek(fifo, buf, n) \
818 __kfifo_uint_must_check_helper( \
819 ({ \
820 	typeof((fifo) + 1) __tmp = (fifo); \
821 	typeof(__tmp->ptr) __buf = (buf); \
822 	unsigned long __n = (n); \
823 	const size_t __recsize = sizeof(*__tmp->rectype); \
824 	struct __kfifo *__kfifo = &__tmp->kfifo; \
825 	(__recsize) ? \
826 	__kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
827 	__kfifo_out_peek(__kfifo, __buf, __n); \
828 }) \
829 )
830 
831 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
832 	size_t esize, gfp_t gfp_mask);
833 
834 extern void __kfifo_free(struct __kfifo *fifo);
835 
836 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
837 	unsigned int size, size_t esize);
838 
839 extern unsigned int __kfifo_in(struct __kfifo *fifo,
840 	const void *buf, unsigned int len);
841 
842 extern unsigned int __kfifo_out(struct __kfifo *fifo,
843 	void *buf, unsigned int len);
844 
845 extern int __kfifo_from_user(struct __kfifo *fifo,
846 	const void __user *from, unsigned long len, unsigned int *copied);
847 
848 extern int __kfifo_to_user(struct __kfifo *fifo,
849 	void __user *to, unsigned long len, unsigned int *copied);
850 
851 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
852 	struct scatterlist *sgl, int nents, unsigned int len);
853 
854 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
855 	struct scatterlist *sgl, int nents, unsigned int len);
856 
857 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
858 	void *buf, unsigned int len);
859 
860 extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
861 	const void *buf, unsigned int len, size_t recsize);
862 
863 extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
864 	void *buf, unsigned int len, size_t recsize);
865 
866 extern int __kfifo_from_user_r(struct __kfifo *fifo,
867 	const void __user *from, unsigned long len, unsigned int *copied,
868 	size_t recsize);
869 
870 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
871 	unsigned long len, unsigned int *copied, size_t recsize);
872 
873 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
874 	struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
875 
876 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
877 	unsigned int len, size_t recsize);
878 
879 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
880 	struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
881 
882 extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
883 
884 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
885 
886 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
887 
888 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
889 	void *buf, unsigned int len, size_t recsize);
890 
891 extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
892 
893 #endif
894