1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_QUEUE_H
8 #define RXE_QUEUE_H
9
10 /* Implements a simple circular buffer that is shared between user
11 * and the driver and can be resized. The requested element size is
12 * rounded up to a power of 2 and the number of elements in the buffer
13 * is also rounded up to a power of 2. Since the queue is empty when
14 * the producer and consumer indices match the maximum capacity of the
15 * queue is one less than the number of element slots.
16 *
17 * Notes:
18 * - The driver indices are always masked off to q->index_mask
19 * before storing so do not need to be checked on reads.
20 * - The user whether user space or kernel is generally
21 * not trusted so its parameters are masked to make sure
22 * they do not access the queue out of bounds on reads.
23 * - The driver indices for queues must not be written
24 * by user so a local copy is used and a shared copy is
25 * stored when the local copy is changed.
26 * - By passing the type in the parameter list separate from q
27 * the compiler can eliminate the switch statement when the
28 * actual queue type is known when the function is called at
29 * compile time.
30 * - These queues are lock free. The user and driver must protect
31 * changes to their end of the queues with locks if more than one
32 * CPU can be accessing it at the same time.
33 */
34
35 /**
36 * enum queue_type - type of queue
37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
38 * read by client. Used by rxe driver only.
39 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
40 * read by rxe driver. Used by rxe driver only.
41 * @QUEUE_TYPE_TO_DRIVER: Queue is written by client and
42 * read by rxe driver. Used by kernel client only.
43 * @QUEUE_TYPE_FROM_DRIVER: Queue is written by rxe driver and
44 * read by client. Used by kernel client only.
45 */
46 enum queue_type {
47 QUEUE_TYPE_TO_CLIENT,
48 QUEUE_TYPE_FROM_CLIENT,
49 QUEUE_TYPE_TO_DRIVER,
50 QUEUE_TYPE_FROM_DRIVER,
51 };
52
53 struct rxe_queue_buf;
54
55 struct rxe_queue {
56 struct rxe_dev *rxe;
57 struct rxe_queue_buf *buf;
58 struct rxe_mmap_info *ip;
59 size_t buf_size;
60 size_t elem_size;
61 unsigned int log2_elem_size;
62 u32 index_mask;
63 enum queue_type type;
64 /* private copy of index for shared queues between
65 * kernel space and user space. Kernel reads and writes
66 * this copy and then replicates to rxe_queue_buf
67 * for read access by user space.
68 */
69 u32 index;
70 };
71
72 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
73 struct ib_udata *udata, struct rxe_queue_buf *buf,
74 size_t buf_size, struct rxe_mmap_info **ip_p);
75
76 void rxe_queue_reset(struct rxe_queue *q);
77
78 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
79 unsigned int elem_size, enum queue_type type);
80
81 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
82 unsigned int elem_size, struct ib_udata *udata,
83 struct mminfo __user *outbuf,
84 spinlock_t *producer_lock, spinlock_t *consumer_lock);
85
86 void rxe_queue_cleanup(struct rxe_queue *queue);
87
queue_next_index(struct rxe_queue * q,int index)88 static inline u32 queue_next_index(struct rxe_queue *q, int index)
89 {
90 return (index + 1) & q->index_mask;
91 }
92
queue_get_producer(const struct rxe_queue * q,enum queue_type type)93 static inline u32 queue_get_producer(const struct rxe_queue *q,
94 enum queue_type type)
95 {
96 u32 prod;
97
98 switch (type) {
99 case QUEUE_TYPE_FROM_CLIENT:
100 /* protect user index */
101 prod = smp_load_acquire(&q->buf->producer_index);
102 break;
103 case QUEUE_TYPE_TO_CLIENT:
104 prod = q->index;
105 break;
106 case QUEUE_TYPE_FROM_DRIVER:
107 /* protect driver index */
108 prod = smp_load_acquire(&q->buf->producer_index);
109 break;
110 case QUEUE_TYPE_TO_DRIVER:
111 prod = q->buf->producer_index;
112 break;
113 }
114
115 return prod;
116 }
117
queue_get_consumer(const struct rxe_queue * q,enum queue_type type)118 static inline u32 queue_get_consumer(const struct rxe_queue *q,
119 enum queue_type type)
120 {
121 u32 cons;
122
123 switch (type) {
124 case QUEUE_TYPE_FROM_CLIENT:
125 cons = q->index;
126 break;
127 case QUEUE_TYPE_TO_CLIENT:
128 /* protect user index */
129 cons = smp_load_acquire(&q->buf->consumer_index);
130 break;
131 case QUEUE_TYPE_FROM_DRIVER:
132 cons = q->buf->consumer_index;
133 break;
134 case QUEUE_TYPE_TO_DRIVER:
135 /* protect driver index */
136 cons = smp_load_acquire(&q->buf->consumer_index);
137 break;
138 }
139
140 return cons;
141 }
142
queue_empty(struct rxe_queue * q,enum queue_type type)143 static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
144 {
145 u32 prod = queue_get_producer(q, type);
146 u32 cons = queue_get_consumer(q, type);
147
148 return ((prod - cons) & q->index_mask) == 0;
149 }
150
queue_full(struct rxe_queue * q,enum queue_type type)151 static inline int queue_full(struct rxe_queue *q, enum queue_type type)
152 {
153 u32 prod = queue_get_producer(q, type);
154 u32 cons = queue_get_consumer(q, type);
155
156 return ((prod + 1 - cons) & q->index_mask) == 0;
157 }
158
queue_count(const struct rxe_queue * q,enum queue_type type)159 static inline u32 queue_count(const struct rxe_queue *q,
160 enum queue_type type)
161 {
162 u32 prod = queue_get_producer(q, type);
163 u32 cons = queue_get_consumer(q, type);
164
165 return (prod - cons) & q->index_mask;
166 }
167
queue_advance_producer(struct rxe_queue * q,enum queue_type type)168 static inline void queue_advance_producer(struct rxe_queue *q,
169 enum queue_type type)
170 {
171 u32 prod;
172
173 switch (type) {
174 case QUEUE_TYPE_FROM_CLIENT:
175 pr_warn("%s: attempt to advance client index\n",
176 __func__);
177 break;
178 case QUEUE_TYPE_TO_CLIENT:
179 prod = q->index;
180 prod = (prod + 1) & q->index_mask;
181 q->index = prod;
182 /* protect user index */
183 smp_store_release(&q->buf->producer_index, prod);
184 break;
185 case QUEUE_TYPE_FROM_DRIVER:
186 pr_warn("%s: attempt to advance driver index\n",
187 __func__);
188 break;
189 case QUEUE_TYPE_TO_DRIVER:
190 prod = q->buf->producer_index;
191 prod = (prod + 1) & q->index_mask;
192 q->buf->producer_index = prod;
193 break;
194 }
195 }
196
queue_advance_consumer(struct rxe_queue * q,enum queue_type type)197 static inline void queue_advance_consumer(struct rxe_queue *q,
198 enum queue_type type)
199 {
200 u32 cons;
201
202 switch (type) {
203 case QUEUE_TYPE_FROM_CLIENT:
204 cons = q->index;
205 cons = (cons + 1) & q->index_mask;
206 q->index = cons;
207 /* protect user index */
208 smp_store_release(&q->buf->consumer_index, cons);
209 break;
210 case QUEUE_TYPE_TO_CLIENT:
211 pr_warn("%s: attempt to advance client index\n",
212 __func__);
213 break;
214 case QUEUE_TYPE_FROM_DRIVER:
215 cons = q->buf->consumer_index;
216 cons = (cons + 1) & q->index_mask;
217 q->buf->consumer_index = cons;
218 break;
219 case QUEUE_TYPE_TO_DRIVER:
220 pr_warn("%s: attempt to advance driver index\n",
221 __func__);
222 break;
223 }
224 }
225
queue_producer_addr(struct rxe_queue * q,enum queue_type type)226 static inline void *queue_producer_addr(struct rxe_queue *q,
227 enum queue_type type)
228 {
229 u32 prod = queue_get_producer(q, type);
230
231 return q->buf->data + (prod << q->log2_elem_size);
232 }
233
queue_consumer_addr(struct rxe_queue * q,enum queue_type type)234 static inline void *queue_consumer_addr(struct rxe_queue *q,
235 enum queue_type type)
236 {
237 u32 cons = queue_get_consumer(q, type);
238
239 return q->buf->data + (cons << q->log2_elem_size);
240 }
241
queue_addr_from_index(struct rxe_queue * q,u32 index)242 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
243 {
244 return q->buf->data + ((index & q->index_mask)
245 << q->log2_elem_size);
246 }
247
queue_index_from_addr(const struct rxe_queue * q,const void * addr)248 static inline u32 queue_index_from_addr(const struct rxe_queue *q,
249 const void *addr)
250 {
251 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
252 & q->index_mask;
253 }
254
queue_head(struct rxe_queue * q,enum queue_type type)255 static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
256 {
257 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
258 }
259
260 #endif /* RXE_QUEUE_H */
261