1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #include "en/params.h"
5 #include "en/txrx.h"
6 #include "en/port.h"
7 #include "en_accel/en_accel.h"
8 #include "en_accel/ipsec.h"
9 #include <net/xdp_sock_drv.h>
10 
mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev * mdev)11 static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev)
12 {
13 	u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size);
14 
15 	return min_page_shift ? : 12;
16 }
17 
mlx5e_mpwrq_page_shift(struct mlx5_core_dev * mdev,struct mlx5e_xsk_param * xsk)18 u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
19 {
20 	u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT;
21 	u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev);
22 
23 	/* Regular RQ uses order-0 pages, the NIC must be able to map them. */
24 	if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift))
25 		min_page_shift = req_page_shift;
26 
27 	return max(req_page_shift, min_page_shift);
28 }
29 
30 enum mlx5e_mpwrq_umr_mode
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev * mdev,struct mlx5e_xsk_param * xsk)31 mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
32 {
33 	/* Different memory management schemes use different mechanisms to map
34 	 * user-mode memory. The stricter guarantees we have, the faster
35 	 * mechanisms we use:
36 	 * 1. MTT - direct mapping in page granularity.
37 	 * 2. KSM - indirect mapping to another MKey to arbitrary addresses, but
38 	 *    all mappings have the same size.
39 	 * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and
40 	 *    mappings can have different sizes.
41 	 */
42 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
43 	bool unaligned = xsk ? xsk->unaligned : false;
44 	bool oversized = false;
45 
46 	if (xsk) {
47 		oversized = xsk->chunk_size < (1 << page_shift);
48 		WARN_ON_ONCE(xsk->chunk_size > (1 << page_shift));
49 	}
50 
51 	/* XSK frame size doesn't match the UMR page size, either because the
52 	 * frame size is not a power of two, or it's smaller than the minimal
53 	 * page size supported by the firmware.
54 	 * It's possible to receive packets bigger than MTU in certain setups.
55 	 * To avoid writing over the XSK frame boundary, the top region of each
56 	 * stride is mapped to a garbage page, resulting in two mappings of
57 	 * different sizes per frame.
58 	 */
59 	if (oversized) {
60 		/* An optimization for frame sizes equal to 3 * power_of_two.
61 		 * 3 KSMs point to the frame, and one KSM points to the garbage
62 		 * page, which works faster than KLM.
63 		 */
64 		if (xsk->chunk_size % 3 == 0 && is_power_of_2(xsk->chunk_size / 3))
65 			return MLX5E_MPWRQ_UMR_MODE_TRIPLE;
66 
67 		return MLX5E_MPWRQ_UMR_MODE_OVERSIZED;
68 	}
69 
70 	/* XSK frames can start at arbitrary unaligned locations, but they all
71 	 * have the same size which is a power of two. It allows to optimize to
72 	 * one KSM per frame.
73 	 */
74 	if (unaligned)
75 		return MLX5E_MPWRQ_UMR_MODE_UNALIGNED;
76 
77 	/* XSK: frames are naturally aligned, MTT can be used.
78 	 * Non-XSK: Allocations happen in units of CPU pages, therefore, the
79 	 * mappings are naturally aligned.
80 	 */
81 	return MLX5E_MPWRQ_UMR_MODE_ALIGNED;
82 }
83 
mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode)84 u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode)
85 {
86 	switch (mode) {
87 	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
88 		return sizeof(struct mlx5_mtt);
89 	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
90 		return sizeof(struct mlx5_ksm);
91 	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
92 		return sizeof(struct mlx5_klm) * 2;
93 	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
94 		return sizeof(struct mlx5_ksm) * 4;
95 	}
96 	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode);
97 	return 0;
98 }
99 
mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)100 u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
101 			  enum mlx5e_mpwrq_umr_mode umr_mode)
102 {
103 	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
104 	u8 max_pages_per_wqe, max_log_mpwqe_size;
105 	u16 max_wqe_size;
106 
107 	/* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */
108 	max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB;
109 	max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe),
110 				       MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size;
111 	max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift;
112 
113 	WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU);
114 
115 	return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ);
116 }
117 
mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)118 u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
119 			     enum mlx5e_mpwrq_umr_mode umr_mode)
120 {
121 	u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
122 	u8 pages_per_wqe;
123 
124 	pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1;
125 
126 	/* Two MTTs are needed to form an octword. The number of MTTs is encoded
127 	 * in octwords in a UMR WQE, so we need at least two to avoid mapping
128 	 * garbage addresses.
129 	 */
130 	if (WARN_ON_ONCE(pages_per_wqe < 2 && umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
131 		pages_per_wqe = 2;
132 
133 	/* Sanity check for further calculations to succeed. */
134 	BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64);
135 	if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE))
136 		return MLX5_MPWRQ_MAX_PAGES_PER_WQE;
137 
138 	return pages_per_wqe;
139 }
140 
mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)141 u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
142 			   enum mlx5e_mpwrq_umr_mode umr_mode)
143 {
144 	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
145 	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
146 	u16 umr_wqe_sz;
147 
148 	umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) +
149 		ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT);
150 
151 	WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK);
152 
153 	return umr_wqe_sz;
154 }
155 
mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)156 u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,
157 			  enum mlx5e_mpwrq_umr_mode umr_mode)
158 {
159 	return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, umr_mode),
160 			    MLX5_SEND_WQE_BB);
161 }
162 
mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)163 u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
164 			    enum mlx5e_mpwrq_umr_mode umr_mode)
165 {
166 	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
167 
168 	/* Add another page as a buffer between WQEs. This page will absorb
169 	 * write overflow by the hardware, when receiving packets larger than
170 	 * MTU. These oversize packets are dropped by the driver at a later
171 	 * stage.
172 	 */
173 	return ALIGN(pages_per_wqe + 1,
174 		     MLX5_SEND_WQE_BB / mlx5e_mpwrq_umr_entry_size(umr_mode));
175 }
176 
mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev * mdev,enum mlx5e_mpwrq_umr_mode umr_mode)177 u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,
178 				enum mlx5e_mpwrq_umr_mode umr_mode)
179 {
180 	/* Same limits apply to KSMs and KLMs. */
181 	u32 klm_limit = min(MLX5E_MAX_RQ_NUM_KSMS,
182 			    1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size));
183 
184 	switch (umr_mode) {
185 	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
186 		return MLX5E_MAX_RQ_NUM_MTTS;
187 	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
188 		return klm_limit;
189 	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
190 		/* Each entry is two KLMs. */
191 		return klm_limit / 2;
192 	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
193 		/* Each entry is four KSMs. */
194 		return klm_limit / 4;
195 	}
196 	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
197 	return 0;
198 }
199 
mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)200 static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift,
201 				      enum mlx5e_mpwrq_umr_mode umr_mode)
202 {
203 	u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, umr_mode);
204 	u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, umr_mode);
205 
206 	return ilog2(max_entries / mtts_per_wqe);
207 }
208 
mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev * mdev,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)209 u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
210 			       enum mlx5e_mpwrq_umr_mode umr_mode)
211 {
212 	return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode) +
213 		mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
214 		MLX5E_ORDER2_MAX_PACKET_MTU;
215 }
216 
mlx5e_get_linear_rq_headroom(struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)217 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
218 				 struct mlx5e_xsk_param *xsk)
219 {
220 	u16 headroom;
221 
222 	if (xsk)
223 		return xsk->headroom;
224 
225 	headroom = NET_IP_ALIGN;
226 	if (params->xdp_prog)
227 		headroom += XDP_PACKET_HEADROOM;
228 	else
229 		headroom += MLX5_RX_HEADROOM;
230 
231 	return headroom;
232 }
233 
mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)234 static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params,
235 				      struct mlx5e_xsk_param *xsk)
236 {
237 	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
238 
239 	return xsk->headroom + hw_mtu;
240 }
241 
mlx5e_rx_get_linear_sz_skb(struct mlx5e_params * params,bool xsk)242 static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk)
243 {
244 	/* SKBs built on XDP_PASS on XSK RQs don't have headroom. */
245 	u16 headroom = xsk ? 0 : mlx5e_get_linear_rq_headroom(params, NULL);
246 	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
247 
248 	return MLX5_SKB_FRAG_SZ(headroom + hw_mtu);
249 }
250 
mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,bool mpwqe)251 static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
252 					 struct mlx5e_params *params,
253 					 struct mlx5e_xsk_param *xsk,
254 					 bool mpwqe)
255 {
256 	/* XSK frames are mapped as individual pages, because frames may come in
257 	 * an arbitrary order from random locations in the UMEM.
258 	 */
259 	if (xsk)
260 		return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
261 
262 	/* XDP in mlx5e doesn't support multiple packets per page. */
263 	if (params->xdp_prog)
264 		return PAGE_SIZE;
265 
266 	return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false));
267 }
268 
mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)269 static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
270 				       struct mlx5e_params *params,
271 				       struct mlx5e_xsk_param *xsk)
272 {
273 	u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);
274 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
275 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
276 
277 	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
278 		order_base_2(linear_stride_sz);
279 }
280 
mlx5e_rx_is_linear_skb(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)281 bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
282 			    struct mlx5e_params *params,
283 			    struct mlx5e_xsk_param *xsk)
284 {
285 	if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE)
286 		return false;
287 
288 	/* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data
289 	 * must fit into a CPU page.
290 	 */
291 	if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE)
292 		return false;
293 
294 	/* XSK frames must be big enough to hold the packet data. */
295 	if (xsk && mlx5e_rx_get_linear_sz_xsk(params, xsk) > xsk->chunk_size)
296 		return false;
297 
298 	return true;
299 }
300 
mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev * mdev,u8 log_stride_sz,u8 log_num_strides,u8 page_shift,enum mlx5e_mpwrq_umr_mode umr_mode)301 static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
302 					  u8 log_stride_sz, u8 log_num_strides,
303 					  u8 page_shift,
304 					  enum mlx5e_mpwrq_umr_mode umr_mode)
305 {
306 	if (log_stride_sz + log_num_strides !=
307 	    mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode))
308 		return false;
309 
310 	if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
311 	    log_stride_sz > MLX5_MPWQE_LOG_STRIDE_SZ_MAX)
312 		return false;
313 
314 	if (log_num_strides > MLX5_MPWQE_LOG_NUM_STRIDES_MAX)
315 		return false;
316 
317 	if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
318 		return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_EXT_BASE;
319 
320 	return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
321 }
322 
mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)323 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
324 				  struct mlx5e_params *params,
325 				  struct mlx5e_xsk_param *xsk)
326 {
327 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
328 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
329 	u8 log_num_strides;
330 	u8 log_stride_sz;
331 	u8 log_wqe_sz;
332 
333 	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk))
334 		return false;
335 
336 	log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
337 	log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
338 
339 	if (log_wqe_sz < log_stride_sz)
340 		return false;
341 
342 	log_num_strides = log_wqe_sz - log_stride_sz;
343 
344 	return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz,
345 					     log_num_strides, page_shift,
346 					     umr_mode);
347 }
348 
mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)349 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
350 			       struct mlx5e_params *params,
351 			       struct mlx5e_xsk_param *xsk)
352 {
353 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
354 	u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
355 
356 	log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
357 	page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
358 	max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);
359 
360 	/* Numbers are unsigned, don't subtract to avoid underflow. */
361 	if (params->log_rq_mtu_frames <
362 	    log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
363 		return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
364 
365 	/* Ethtool's rx_max_pending is calculated for regular RQ, that uses
366 	 * pages of PAGE_SIZE. Max length of an XSK RQ might differ if it uses a
367 	 * frame size not equal to PAGE_SIZE.
368 	 * A stricter condition is checked in mlx5e_mpwrq_validate_xsk, WARN on
369 	 * unexpected failure.
370 	 */
371 	if (WARN_ON_ONCE(params->log_rq_mtu_frames > log_pkts_per_wqe + max_log_rq_size))
372 		return max_log_rq_size;
373 
374 	return params->log_rq_mtu_frames - log_pkts_per_wqe;
375 }
376 
mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params)377 u8 mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev *mdev,
378 				      struct mlx5e_params *params)
379 {
380 	return order_base_2(DIV_ROUND_UP(MLX5E_RX_MAX_HEAD, MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE));
381 }
382 
mlx5e_shampo_get_log_rsrv_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params)383 u8 mlx5e_shampo_get_log_rsrv_size(struct mlx5_core_dev *mdev,
384 				  struct mlx5e_params *params)
385 {
386 	return order_base_2(MLX5E_SHAMPO_WQ_RESRV_SIZE / MLX5E_SHAMPO_WQ_BASE_RESRV_SIZE);
387 }
388 
mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5_core_dev * mdev,struct mlx5e_params * params)389 u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5_core_dev *mdev,
390 				     struct mlx5e_params *params)
391 {
392 	u32 resrv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) *
393 			 PAGE_SIZE;
394 
395 	return order_base_2(DIV_ROUND_UP(resrv_size, params->sw_mtu));
396 }
397 
mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)398 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
399 				   struct mlx5e_params *params,
400 				   struct mlx5e_xsk_param *xsk)
401 {
402 	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
403 		return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
404 
405 	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
406 }
407 
mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)408 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
409 				   struct mlx5e_params *params,
410 				   struct mlx5e_xsk_param *xsk)
411 {
412 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
413 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
414 
415 	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
416 		mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
417 }
418 
mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz)419 u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz)
420 {
421 #define UMR_WQE_BULK (2)
422 	return min_t(unsigned int, UMR_WQE_BULK, wq_sz / 2 - 1);
423 }
424 
mlx5e_get_rq_headroom(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)425 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
426 			  struct mlx5e_params *params,
427 			  struct mlx5e_xsk_param *xsk)
428 {
429 	u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
430 
431 	if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
432 		return linear_headroom;
433 
434 	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
435 		return linear_headroom;
436 
437 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
438 		return linear_headroom;
439 
440 	return 0;
441 }
442 
mlx5e_calc_sq_stop_room(struct mlx5_core_dev * mdev,struct mlx5e_params * params)443 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
444 {
445 	bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
446 	u16 stop_room;
447 
448 	stop_room  = mlx5e_ktls_get_stop_room(mdev, params);
449 	stop_room += mlx5e_stop_room_for_max_wqe(mdev);
450 	if (is_mpwqe)
451 		/* A MPWQE can take up to the maximum cacheline-aligned WQE +
452 		 * all the normal stop room can be taken if a new packet breaks
453 		 * the active MPWQE session and allocates its WQEs right away.
454 		 */
455 		stop_room += mlx5e_stop_room_for_mpwqe(mdev);
456 
457 	return stop_room;
458 }
459 
mlx5e_validate_params(struct mlx5_core_dev * mdev,struct mlx5e_params * params)460 int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
461 {
462 	size_t sq_size = 1 << params->log_sq_size;
463 	u16 stop_room;
464 
465 	stop_room = mlx5e_calc_sq_stop_room(mdev, params);
466 	if (stop_room >= sq_size) {
467 		mlx5_core_err(mdev, "Stop room %u is bigger than the SQ size %zu\n",
468 			      stop_room, sq_size);
469 		return -EINVAL;
470 	}
471 
472 	return 0;
473 }
474 
mlx5e_get_def_tx_moderation(u8 cq_period_mode)475 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
476 {
477 	struct dim_cq_moder moder = {};
478 
479 	moder.cq_period_mode = cq_period_mode;
480 	moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
481 	moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
482 	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
483 		moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
484 
485 	return moder;
486 }
487 
mlx5e_get_def_rx_moderation(u8 cq_period_mode)488 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
489 {
490 	struct dim_cq_moder moder = {};
491 
492 	moder.cq_period_mode = cq_period_mode;
493 	moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
494 	moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
495 	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
496 		moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
497 
498 	return moder;
499 }
500 
mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)501 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
502 {
503 	return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
504 		DIM_CQ_PERIOD_MODE_START_FROM_CQE :
505 		DIM_CQ_PERIOD_MODE_START_FROM_EQE;
506 }
507 
mlx5e_reset_tx_moderation(struct mlx5e_params * params,u8 cq_period_mode)508 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
509 {
510 	if (params->tx_dim_enabled) {
511 		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
512 
513 		params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
514 	} else {
515 		params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
516 	}
517 }
518 
mlx5e_reset_rx_moderation(struct mlx5e_params * params,u8 cq_period_mode)519 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
520 {
521 	if (params->rx_dim_enabled) {
522 		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
523 
524 		params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
525 	} else {
526 		params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
527 	}
528 }
529 
mlx5e_set_tx_cq_mode_params(struct mlx5e_params * params,u8 cq_period_mode)530 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
531 {
532 	mlx5e_reset_tx_moderation(params, cq_period_mode);
533 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
534 			params->tx_cq_moderation.cq_period_mode ==
535 				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
536 }
537 
mlx5e_set_rx_cq_mode_params(struct mlx5e_params * params,u8 cq_period_mode)538 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
539 {
540 	mlx5e_reset_rx_moderation(params, cq_period_mode);
541 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
542 			params->rx_cq_moderation.cq_period_mode ==
543 				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
544 }
545 
slow_pci_heuristic(struct mlx5_core_dev * mdev)546 bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
547 {
548 	u32 link_speed = 0;
549 	u32 pci_bw = 0;
550 
551 	mlx5e_port_max_linkspeed(mdev, &link_speed);
552 	pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
553 	mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
554 			   link_speed, pci_bw);
555 
556 #define MLX5E_SLOW_PCI_RATIO (2)
557 
558 	return link_speed && pci_bw &&
559 		link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
560 }
561 
mlx5e_mpwrq_validate_regular(struct mlx5_core_dev * mdev,struct mlx5e_params * params)562 int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
563 {
564 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, NULL);
565 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL);
566 
567 	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
568 		return -EOPNOTSUPP;
569 
570 	if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
571 		return -EINVAL;
572 
573 	return 0;
574 }
575 
mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)576 int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
577 			     struct mlx5e_xsk_param *xsk)
578 {
579 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
580 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
581 	bool unaligned = xsk ? xsk->unaligned : false;
582 	u16 max_mtu_pkts;
583 
584 	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
585 		return -EOPNOTSUPP;
586 
587 	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
588 		return -EINVAL;
589 
590 	/* Current RQ length is too big for the given frame size, the
591 	 * needed number of WQEs exceeds the maximum.
592 	 */
593 	max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
594 			     mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, unaligned));
595 	if (params->log_rq_mtu_frames > max_mtu_pkts) {
596 		mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
597 			      1 << params->log_rq_mtu_frames, xsk->chunk_size);
598 		return -EINVAL;
599 	}
600 
601 	return 0;
602 }
603 
mlx5e_init_rq_type_params(struct mlx5_core_dev * mdev,struct mlx5e_params * params)604 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
605 			       struct mlx5e_params *params)
606 {
607 	params->log_rq_mtu_frames = is_kdump_kernel() ?
608 		MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
609 		MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
610 
611 	mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
612 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
613 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
614 		       BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) :
615 		       BIT(params->log_rq_mtu_frames),
616 		       BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)),
617 		       MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
618 }
619 
mlx5e_set_rq_type(struct mlx5_core_dev * mdev,struct mlx5e_params * params)620 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
621 {
622 	params->rq_wq_type = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
623 		MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
624 		MLX5_WQ_TYPE_CYCLIC;
625 }
626 
mlx5e_build_rq_params(struct mlx5_core_dev * mdev,struct mlx5e_params * params)627 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
628 			   struct mlx5e_params *params)
629 {
630 	/* Prefer Striding RQ, unless any of the following holds:
631 	 * - Striding RQ configuration is not possible/supported.
632 	 * - CQE compression is ON, and stride_index mini_cqe layout is not supported.
633 	 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
634 	 *
635 	 * No XSK params: checking the availability of striding RQ in general.
636 	 */
637 	if ((!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) ||
638 	     MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) &&
639 	    !mlx5e_mpwrq_validate_regular(mdev, params) &&
640 	    (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ||
641 	     !mlx5e_rx_is_linear_skb(mdev, params, NULL)))
642 		MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
643 	mlx5e_set_rq_type(mdev, params);
644 	mlx5e_init_rq_type_params(mdev, params);
645 }
646 
647 /* Build queue parameters */
648 
mlx5e_build_create_cq_param(struct mlx5e_create_cq_param * ccp,struct mlx5e_channel * c)649 void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c)
650 {
651 	*ccp = (struct mlx5e_create_cq_param) {
652 		.napi = &c->napi,
653 		.ch_stats = c->stats,
654 		.node = cpu_to_node(c->cpu),
655 		.ix = c->ix,
656 	};
657 }
658 
mlx5e_max_nonlinear_mtu(int first_frag_size,int frag_size,bool xdp)659 static int mlx5e_max_nonlinear_mtu(int first_frag_size, int frag_size, bool xdp)
660 {
661 	if (xdp)
662 		/* XDP requires all fragments to be of the same size. */
663 		return first_frag_size + (MLX5E_MAX_RX_FRAGS - 1) * frag_size;
664 
665 	/* Optimization for small packets: the last fragment is bigger than the others. */
666 	return first_frag_size + (MLX5E_MAX_RX_FRAGS - 2) * frag_size + PAGE_SIZE;
667 }
668 
669 #define DEFAULT_FRAG_SIZE (2048)
670 
mlx5e_build_rq_frags_info(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5e_rq_frags_info * info)671 static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
672 				     struct mlx5e_params *params,
673 				     struct mlx5e_xsk_param *xsk,
674 				     struct mlx5e_rq_frags_info *info)
675 {
676 	u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
677 	int frag_size_max = DEFAULT_FRAG_SIZE;
678 	int first_frag_size_max;
679 	u32 buf_size = 0;
680 	u16 headroom;
681 	int max_mtu;
682 	int i;
683 
684 	if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
685 		int frag_stride;
686 
687 		frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false);
688 
689 		info->arr[0].frag_size = byte_count;
690 		info->arr[0].frag_stride = frag_stride;
691 		info->num_frags = 1;
692 
693 		/* N WQEs share the same page, N = PAGE_SIZE / frag_stride. The
694 		 * first WQE in the page is responsible for allocation of this
695 		 * page, this WQE's index is k*N. If WQEs [k*N+1; k*N+N-1] are
696 		 * still not completed, the allocation must stop before k*N.
697 		 */
698 		info->wqe_index_mask = (PAGE_SIZE / frag_stride) - 1;
699 
700 		goto out;
701 	}
702 
703 	headroom = mlx5e_get_linear_rq_headroom(params, xsk);
704 	first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
705 
706 	max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max,
707 					  params->xdp_prog);
708 	if (byte_count > max_mtu || params->xdp_prog) {
709 		frag_size_max = PAGE_SIZE;
710 		first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
711 
712 		max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max,
713 						  params->xdp_prog);
714 		if (byte_count > max_mtu) {
715 			mlx5_core_err(mdev, "MTU %u is too big for non-linear legacy RQ (max %d)\n",
716 				      params->sw_mtu, max_mtu);
717 			return -EINVAL;
718 		}
719 	}
720 
721 	i = 0;
722 	while (buf_size < byte_count) {
723 		int frag_size = byte_count - buf_size;
724 
725 		if (i == 0)
726 			frag_size = min(frag_size, first_frag_size_max);
727 		else if (i < MLX5E_MAX_RX_FRAGS - 1)
728 			frag_size = min(frag_size, frag_size_max);
729 
730 		info->arr[i].frag_size = frag_size;
731 		buf_size += frag_size;
732 
733 		if (params->xdp_prog) {
734 			/* XDP multi buffer expects fragments of the same size. */
735 			info->arr[i].frag_stride = frag_size_max;
736 		} else {
737 			if (i == 0) {
738 				/* Ensure that headroom and tailroom are included. */
739 				frag_size += headroom;
740 				frag_size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
741 			}
742 			info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
743 		}
744 
745 		i++;
746 	}
747 	info->num_frags = i;
748 
749 	/* The last fragment of WQE with index 2*N may share the page with the
750 	 * first fragment of WQE with index 2*N+1 in certain cases. If WQE 2*N+1
751 	 * is not completed yet, WQE 2*N must not be allocated, as it's
752 	 * responsible for allocating a new page.
753 	 */
754 	if (frag_size_max == PAGE_SIZE) {
755 		/* No WQE can start in the middle of a page. */
756 		info->wqe_index_mask = 0;
757 	} else {
758 		/* PAGE_SIZEs starting from 8192 don't use 2K-sized fragments,
759 		 * because there would be more than MLX5E_MAX_RX_FRAGS of them.
760 		 */
761 		WARN_ON(PAGE_SIZE != 2 * DEFAULT_FRAG_SIZE);
762 
763 		/* Odd number of fragments allows to pack the last fragment of
764 		 * the previous WQE and the first fragment of the next WQE into
765 		 * the same page.
766 		 * As long as DEFAULT_FRAG_SIZE is 2048, and MLX5E_MAX_RX_FRAGS
767 		 * is 4, the last fragment can be bigger than the rest only if
768 		 * it's the fourth one, so WQEs consisting of 3 fragments will
769 		 * always share a page.
770 		 * When a page is shared, WQE bulk size is 2, otherwise just 1.
771 		 */
772 		info->wqe_index_mask = info->num_frags % 2;
773 	}
774 
775 out:
776 	/* Bulking optimization to skip allocation until at least 8 WQEs can be
777 	 * allocated in a row. At the same time, never start allocation when
778 	 * the page is still used by older WQEs.
779 	 */
780 	info->wqe_bulk = max_t(u8, info->wqe_index_mask + 1, 8);
781 
782 	info->log_num_frags = order_base_2(info->num_frags);
783 
784 	return 0;
785 }
786 
mlx5e_get_rqwq_log_stride(u8 wq_type,int ndsegs)787 static u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
788 {
789 	int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
790 
791 	switch (wq_type) {
792 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
793 		sz += sizeof(struct mlx5e_rx_wqe_ll);
794 		break;
795 	default: /* MLX5_WQ_TYPE_CYCLIC */
796 		sz += sizeof(struct mlx5e_rx_wqe_cyc);
797 	}
798 
799 	return order_base_2(sz);
800 }
801 
mlx5e_build_common_cq_param(struct mlx5_core_dev * mdev,struct mlx5e_cq_param * param)802 static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
803 					struct mlx5e_cq_param *param)
804 {
805 	void *cqc = param->cqc;
806 
807 	MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
808 	if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
809 		MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
810 }
811 
mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)812 static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
813 					struct mlx5e_params *params,
814 					struct mlx5e_xsk_param *xsk)
815 {
816 	int rsrv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) * PAGE_SIZE;
817 	u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
818 	int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params));
819 	u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
820 	int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
821 	int wqe_size = BIT(log_stride_sz) * num_strides;
822 
823 	/* +1 is for the case that the pkt_per_rsrv dont consume the reservation
824 	 * so we get a filler cqe for the rest of the reservation.
825 	 */
826 	return order_base_2((wqe_size / rsrv_size) * wq_size * (pkt_per_rsrv + 1));
827 }
828 
mlx5e_build_rx_cq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5e_cq_param * param)829 static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
830 				    struct mlx5e_params *params,
831 				    struct mlx5e_xsk_param *xsk,
832 				    struct mlx5e_cq_param *param)
833 {
834 	bool hw_stridx = false;
835 	void *cqc = param->cqc;
836 	u8 log_cq_size;
837 
838 	switch (params->rq_wq_type) {
839 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
840 		hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
841 		if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
842 			log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk);
843 		else
844 			log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) +
845 				mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
846 		break;
847 	default: /* MLX5_WQ_TYPE_CYCLIC */
848 		log_cq_size = params->log_rq_mtu_frames;
849 	}
850 
851 	MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
852 	if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
853 		MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ?
854 			 MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM);
855 		MLX5_SET(cqc, cqc, cqe_comp_en, 1);
856 	}
857 
858 	mlx5e_build_common_cq_param(mdev, param);
859 	param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
860 }
861 
rq_end_pad_mode(struct mlx5_core_dev * mdev,struct mlx5e_params * params)862 static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
863 {
864 	bool lro_en = params->packet_merge.type == MLX5E_PACKET_MERGE_LRO;
865 	bool ro = pcie_relaxed_ordering_enabled(mdev->pdev) &&
866 		MLX5_CAP_GEN(mdev, relaxed_ordering_write);
867 
868 	return ro && lro_en ?
869 		MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
870 }
871 
mlx5e_build_rq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,u16 q_counter,struct mlx5e_rq_param * param)872 int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
873 			 struct mlx5e_params *params,
874 			 struct mlx5e_xsk_param *xsk,
875 			 u16 q_counter,
876 			 struct mlx5e_rq_param *param)
877 {
878 	void *rqc = param->rqc;
879 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
880 	int ndsegs = 1;
881 	int err;
882 
883 	switch (params->rq_wq_type) {
884 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
885 		u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
886 		u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
887 		enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
888 		u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
889 
890 		if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
891 						   log_wqe_num_of_strides,
892 						   page_shift, umr_mode)) {
893 			mlx5_core_err(mdev,
894 				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
895 				      log_wqe_stride_size, log_wqe_num_of_strides,
896 				      umr_mode);
897 			return -EINVAL;
898 		}
899 
900 		MLX5_SET(wq, wq, log_wqe_num_of_strides,
901 			 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
902 		MLX5_SET(wq, wq, log_wqe_stride_size,
903 			 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
904 		MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
905 		if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
906 			MLX5_SET(wq, wq, shampo_enable, true);
907 			MLX5_SET(wq, wq, log_reservation_size,
908 				 mlx5e_shampo_get_log_rsrv_size(mdev, params));
909 			MLX5_SET(wq, wq,
910 				 log_max_num_of_packets_per_reservation,
911 				 mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params));
912 			MLX5_SET(wq, wq, log_headers_entry_size,
913 				 mlx5e_shampo_get_log_hd_entry_size(mdev, params));
914 			MLX5_SET(rqc, rqc, reservation_timeout,
915 				 params->packet_merge.timeout);
916 			MLX5_SET(rqc, rqc, shampo_match_criteria_type,
917 				 params->packet_merge.shampo.match_criteria_type);
918 			MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
919 				 params->packet_merge.shampo.alignment_granularity);
920 		}
921 		break;
922 	}
923 	default: /* MLX5_WQ_TYPE_CYCLIC */
924 		MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
925 		err = mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info);
926 		if (err)
927 			return err;
928 		ndsegs = param->frags_info.num_frags;
929 	}
930 
931 	MLX5_SET(wq, wq, wq_type,          params->rq_wq_type);
932 	MLX5_SET(wq, wq, end_padding_mode, rq_end_pad_mode(mdev, params));
933 	MLX5_SET(wq, wq, log_wq_stride,
934 		 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
935 	MLX5_SET(wq, wq, pd,               mdev->mlx5e_res.hw_objs.pdn);
936 	MLX5_SET(rqc, rqc, counter_set_id, q_counter);
937 	MLX5_SET(rqc, rqc, vsd,            params->vlan_strip_disable);
938 	MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
939 
940 	param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
941 	mlx5e_build_rx_cq_param(mdev, params, xsk, &param->cqp);
942 
943 	return 0;
944 }
945 
mlx5e_build_drop_rq_param(struct mlx5_core_dev * mdev,u16 q_counter,struct mlx5e_rq_param * param)946 void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
947 			       u16 q_counter,
948 			       struct mlx5e_rq_param *param)
949 {
950 	void *rqc = param->rqc;
951 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
952 
953 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
954 	MLX5_SET(wq, wq, log_wq_stride,
955 		 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
956 	MLX5_SET(rqc, rqc, counter_set_id, q_counter);
957 
958 	param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
959 }
960 
mlx5e_build_tx_cq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_cq_param * param)961 void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
962 			     struct mlx5e_params *params,
963 			     struct mlx5e_cq_param *param)
964 {
965 	void *cqc = param->cqc;
966 
967 	MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
968 
969 	mlx5e_build_common_cq_param(mdev, param);
970 	param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
971 }
972 
mlx5e_build_sq_param_common(struct mlx5_core_dev * mdev,struct mlx5e_sq_param * param)973 void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev,
974 				 struct mlx5e_sq_param *param)
975 {
976 	void *sqc = param->sqc;
977 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
978 
979 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
980 	MLX5_SET(wq, wq, pd,            mdev->mlx5e_res.hw_objs.pdn);
981 
982 	param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
983 }
984 
mlx5e_build_sq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_sq_param * param)985 void mlx5e_build_sq_param(struct mlx5_core_dev *mdev,
986 			  struct mlx5e_params *params,
987 			  struct mlx5e_sq_param *param)
988 {
989 	void *sqc = param->sqc;
990 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
991 	bool allow_swp;
992 
993 	allow_swp =
994 		mlx5_geneve_tx_allowed(mdev) || !!mlx5_ipsec_device_caps(mdev);
995 	mlx5e_build_sq_param_common(mdev, param);
996 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
997 	MLX5_SET(sqc, sqc, allow_swp, allow_swp);
998 	param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
999 	param->stop_room = mlx5e_calc_sq_stop_room(mdev, params);
1000 	mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
1001 }
1002 
mlx5e_build_ico_cq_param(struct mlx5_core_dev * mdev,u8 log_wq_size,struct mlx5e_cq_param * param)1003 static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev,
1004 				     u8 log_wq_size,
1005 				     struct mlx5e_cq_param *param)
1006 {
1007 	void *cqc = param->cqc;
1008 
1009 	MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1010 
1011 	mlx5e_build_common_cq_param(mdev, param);
1012 
1013 	param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1014 }
1015 
1016 /* This function calculates the maximum number of headers entries that are needed
1017  * per WQE, the formula is based on the size of the reservations and the
1018  * restriction we have about max packets for reservation that is equal to max
1019  * headers per reservation.
1020  */
mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_param * rq_param)1021 u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev,
1022 			    struct mlx5e_params *params,
1023 			    struct mlx5e_rq_param *rq_param)
1024 {
1025 	int resv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) * PAGE_SIZE;
1026 	u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, NULL));
1027 	int pkt_per_resv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params));
1028 	u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL);
1029 	int wqe_size = BIT(log_stride_sz) * num_strides;
1030 	u32 hd_per_wqe;
1031 
1032 	/* Assumption: hd_per_wqe % 8 == 0. */
1033 	hd_per_wqe = (wqe_size / resv_size) * pkt_per_resv;
1034 	mlx5_core_dbg(mdev, "%s hd_per_wqe = %d rsrv_size = %d wqe_size = %d pkt_per_resv = %d\n",
1035 		      __func__, hd_per_wqe, resv_size, wqe_size, pkt_per_resv);
1036 	return hd_per_wqe;
1037 }
1038 
1039 /* This function calculates the maximum number of headers entries that are needed
1040  * for the WQ, this value is uesed to allocate the header buffer in HW, thus
1041  * must be a pow of 2.
1042  */
mlx5e_shampo_hd_per_wq(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_param * rq_param)1043 u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev,
1044 			   struct mlx5e_params *params,
1045 			   struct mlx5e_rq_param *rq_param)
1046 {
1047 	void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
1048 	int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
1049 	u32 hd_per_wqe, hd_per_wq;
1050 
1051 	hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param);
1052 	hd_per_wq = roundup_pow_of_two(hd_per_wqe * wq_size);
1053 	return hd_per_wq;
1054 }
1055 
mlx5e_shampo_icosq_sz(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_param * rq_param)1056 static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev,
1057 				 struct mlx5e_params *params,
1058 				 struct mlx5e_rq_param *rq_param)
1059 {
1060 	int max_num_of_umr_per_wqe, max_hd_per_wqe, max_klm_per_umr, rest;
1061 	void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
1062 	int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
1063 	u32 wqebbs;
1064 
1065 	max_klm_per_umr = MLX5E_MAX_KLM_PER_WQE(mdev);
1066 	max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param);
1067 	max_num_of_umr_per_wqe = max_hd_per_wqe / max_klm_per_umr;
1068 	rest = max_hd_per_wqe % max_klm_per_umr;
1069 	wqebbs = MLX5E_KLM_UMR_WQEBBS(max_klm_per_umr) * max_num_of_umr_per_wqe;
1070 	if (rest)
1071 		wqebbs += MLX5E_KLM_UMR_WQEBBS(rest);
1072 	wqebbs *= wq_size;
1073 	return wqebbs;
1074 }
1075 
mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)1076 static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
1077 					struct mlx5e_params *params,
1078 					struct mlx5e_xsk_param *xsk)
1079 {
1080 	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
1081 	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
1082 	u8 umr_wqebbs;
1083 
1084 	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
1085 
1086 	return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
1087 }
1088 
mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_param * rqp)1089 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
1090 				      struct mlx5e_params *params,
1091 				      struct mlx5e_rq_param *rqp)
1092 {
1093 	u32 wqebbs, total_pages, useful_space;
1094 
1095 	/* MLX5_WQ_TYPE_CYCLIC */
1096 	if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
1097 		return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1098 
1099 	/* UMR WQEs for the regular RQ. */
1100 	wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
1101 
1102 	/* If XDP program is attached, XSK may be turned on at any time without
1103 	 * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
1104 	 * both regular RQ and XSK RQ.
1105 	 *
1106 	 * XSK uses different values of page_shift, and the total number of UMR
1107 	 * WQEBBs depends on it. This dependency is complex and not monotonic,
1108 	 * especially taking into consideration that some of the parameters come
1109 	 * from capabilities. Hence, we have to try all valid values of XSK
1110 	 * frame size (and page_shift) to find the maximum.
1111 	 */
1112 	if (params->xdp_prog) {
1113 		u32 max_xsk_wqebbs = 0;
1114 		u8 frame_shift;
1115 
1116 		for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
1117 		     frame_shift <= PAGE_SHIFT; frame_shift++) {
1118 			/* The headroom doesn't affect the calculation. */
1119 			struct mlx5e_xsk_param xsk = {
1120 				.chunk_size = 1 << frame_shift,
1121 				.unaligned = false,
1122 			};
1123 
1124 			/* XSK aligned mode. */
1125 			max_xsk_wqebbs = max(max_xsk_wqebbs,
1126 				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1127 
1128 			/* XSK unaligned mode, frame size is a power of two. */
1129 			xsk.unaligned = true;
1130 			max_xsk_wqebbs = max(max_xsk_wqebbs,
1131 				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1132 
1133 			/* XSK unaligned mode, frame size is not equal to stride size. */
1134 			xsk.chunk_size -= 1;
1135 			max_xsk_wqebbs = max(max_xsk_wqebbs,
1136 				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1137 
1138 			/* XSK unaligned mode, frame size is a triple power of two. */
1139 			xsk.chunk_size = (1 << frame_shift) / 4 * 3;
1140 			max_xsk_wqebbs = max(max_xsk_wqebbs,
1141 				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1142 		}
1143 
1144 		wqebbs += max_xsk_wqebbs;
1145 	}
1146 
1147 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1148 		wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp);
1149 
1150 	/* UMR WQEs don't cross the page boundary, they are padded with NOPs.
1151 	 * This padding is always smaller than the max WQE size. That gives us
1152 	 * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes
1153 	 * per page. The number of pages is estimated as the total size of WQEs
1154 	 * divided by the useful space in page, rounding up. If some WQEs don't
1155 	 * fully fit into the useful space, they can occupy part of the padding,
1156 	 * which proves this estimation to be correct (reserve enough space).
1157 	 */
1158 	useful_space = PAGE_SIZE - mlx5e_get_max_sq_wqebbs(mdev) + MLX5_SEND_WQE_BB;
1159 	total_pages = DIV_ROUND_UP(wqebbs * MLX5_SEND_WQE_BB, useful_space);
1160 	wqebbs = total_pages * (PAGE_SIZE / MLX5_SEND_WQE_BB);
1161 
1162 	return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs));
1163 }
1164 
mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev * mdev)1165 static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev)
1166 {
1167 	if (mlx5e_is_ktls_rx(mdev))
1168 		return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
1169 
1170 	return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1171 }
1172 
mlx5e_build_icosq_param(struct mlx5_core_dev * mdev,u8 log_wq_size,struct mlx5e_sq_param * param)1173 static void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
1174 				    u8 log_wq_size,
1175 				    struct mlx5e_sq_param *param)
1176 {
1177 	void *sqc = param->sqc;
1178 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1179 
1180 	mlx5e_build_sq_param_common(mdev, param);
1181 
1182 	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1183 	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
1184 	mlx5e_build_ico_cq_param(mdev, log_wq_size, &param->cqp);
1185 }
1186 
mlx5e_build_async_icosq_param(struct mlx5_core_dev * mdev,u8 log_wq_size,struct mlx5e_sq_param * param)1187 static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev,
1188 					  u8 log_wq_size,
1189 					  struct mlx5e_sq_param *param)
1190 {
1191 	void *sqc = param->sqc;
1192 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1193 
1194 	mlx5e_build_sq_param_common(mdev, param);
1195 	param->stop_room = mlx5e_stop_room_for_wqe(mdev, 1); /* for XSK NOP */
1196 	param->is_tls = mlx5e_is_ktls_rx(mdev);
1197 	if (param->is_tls)
1198 		param->stop_room += mlx5e_stop_room_for_wqe(mdev, 1); /* for TLS RX resync NOP */
1199 	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
1200 	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1201 	mlx5e_build_ico_cq_param(mdev, log_wq_size, &param->cqp);
1202 }
1203 
mlx5e_build_xdpsq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5e_sq_param * param)1204 void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
1205 			     struct mlx5e_params *params,
1206 			     struct mlx5e_xsk_param *xsk,
1207 			     struct mlx5e_sq_param *param)
1208 {
1209 	void *sqc = param->sqc;
1210 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1211 
1212 	mlx5e_build_sq_param_common(mdev, param);
1213 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
1214 	param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
1215 	param->is_xdp_mb = !mlx5e_rx_is_linear_skb(mdev, params, xsk);
1216 	mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
1217 }
1218 
mlx5e_build_channel_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 q_counter,struct mlx5e_channel_param * cparam)1219 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
1220 			      struct mlx5e_params *params,
1221 			      u16 q_counter,
1222 			      struct mlx5e_channel_param *cparam)
1223 {
1224 	u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
1225 	int err;
1226 
1227 	err = mlx5e_build_rq_param(mdev, params, NULL, q_counter, &cparam->rq);
1228 	if (err)
1229 		return err;
1230 
1231 	icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(mdev, params, &cparam->rq);
1232 	async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev);
1233 
1234 	mlx5e_build_sq_param(mdev, params, &cparam->txq_sq);
1235 	mlx5e_build_xdpsq_param(mdev, params, NULL, &cparam->xdp_sq);
1236 	mlx5e_build_icosq_param(mdev, icosq_log_wq_sz, &cparam->icosq);
1237 	mlx5e_build_async_icosq_param(mdev, async_icosq_log_wq_sz, &cparam->async_icosq);
1238 
1239 	return 0;
1240 }
1241