1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */
3
4 #include <net/xdp_sock_drv.h>
5 #include "pool.h"
6 #include "setup.h"
7 #include "en/params.h"
8
mlx5e_xsk_map_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)9 static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv,
10 struct xsk_buff_pool *pool)
11 {
12 struct device *dev = mlx5_core_dma_dev(priv->mdev);
13
14 return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
15 }
16
mlx5e_xsk_unmap_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)17 static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
18 struct xsk_buff_pool *pool)
19 {
20 return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
21 }
22
mlx5e_xsk_get_pools(struct mlx5e_xsk * xsk)23 static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
24 {
25 if (!xsk->pools) {
26 xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
27 sizeof(*xsk->pools), GFP_KERNEL);
28 if (unlikely(!xsk->pools))
29 return -ENOMEM;
30 }
31
32 xsk->refcnt++;
33 xsk->ever_used = true;
34
35 return 0;
36 }
37
mlx5e_xsk_put_pools(struct mlx5e_xsk * xsk)38 static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
39 {
40 if (!--xsk->refcnt) {
41 kfree(xsk->pools);
42 xsk->pools = NULL;
43 }
44 }
45
mlx5e_xsk_add_pool(struct mlx5e_xsk * xsk,struct xsk_buff_pool * pool,u16 ix)46 static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix)
47 {
48 int err;
49
50 err = mlx5e_xsk_get_pools(xsk);
51 if (unlikely(err))
52 return err;
53
54 xsk->pools[ix] = pool;
55 return 0;
56 }
57
mlx5e_xsk_remove_pool(struct mlx5e_xsk * xsk,u16 ix)58 static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
59 {
60 xsk->pools[ix] = NULL;
61
62 mlx5e_xsk_put_pools(xsk);
63 }
64
mlx5e_xsk_is_pool_sane(struct xsk_buff_pool * pool)65 static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
66 {
67 return xsk_pool_get_headroom(pool) <= 0xffff &&
68 xsk_pool_get_chunk_size(pool) <= 0xffff;
69 }
70
mlx5e_build_xsk_param(struct xsk_buff_pool * pool,struct mlx5e_xsk_param * xsk)71 void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
72 {
73 xsk->headroom = xsk_pool_get_headroom(pool);
74 xsk->chunk_size = xsk_pool_get_chunk_size(pool);
75 }
76
mlx5e_xsk_enable_locked(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)77 static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
78 struct xsk_buff_pool *pool, u16 ix)
79 {
80 struct mlx5e_params *params = &priv->channels.params;
81 struct mlx5e_xsk_param xsk;
82 struct mlx5e_channel *c;
83 int err;
84
85 if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
86 return -EBUSY;
87
88 if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
89 return -EINVAL;
90
91 err = mlx5e_xsk_map_pool(priv, pool);
92 if (unlikely(err))
93 return err;
94
95 err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
96 if (unlikely(err))
97 goto err_unmap_pool;
98
99 mlx5e_build_xsk_param(pool, &xsk);
100
101 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
102 /* XSK objects will be created on open. */
103 goto validate_closed;
104 }
105
106 if (!params->xdp_prog) {
107 /* XSK objects will be created when an XDP program is set,
108 * and the channels are reopened.
109 */
110 goto validate_closed;
111 }
112
113 c = priv->channels.c[ix];
114
115 err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
116 if (unlikely(err))
117 goto err_remove_pool;
118
119 mlx5e_activate_xsk(c);
120 mlx5e_trigger_napi_icosq(c);
121
122 /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
123 * any Fill Ring entries at the setup stage.
124 */
125
126 err = mlx5e_rx_res_xsk_activate(priv->rx_res, &priv->channels, ix);
127 if (unlikely(err))
128 goto err_deactivate;
129
130 return 0;
131
132 err_deactivate:
133 mlx5e_deactivate_xsk(c);
134 mlx5e_close_xsk(c);
135
136 err_remove_pool:
137 mlx5e_xsk_remove_pool(&priv->xsk, ix);
138
139 err_unmap_pool:
140 mlx5e_xsk_unmap_pool(priv, pool);
141
142 return err;
143
144 validate_closed:
145 /* Check the configuration in advance, rather than fail at a later stage
146 * (in mlx5e_xdp_set or on open) and end up with no channels.
147 */
148 if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
149 err = -EINVAL;
150 goto err_remove_pool;
151 }
152
153 return 0;
154 }
155
mlx5e_xsk_disable_locked(struct mlx5e_priv * priv,u16 ix)156 static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
157 {
158 struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(&priv->channels.params,
159 &priv->xsk, ix);
160 struct mlx5e_channel *c;
161
162 if (unlikely(!pool))
163 return -EINVAL;
164
165 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
166 goto remove_pool;
167
168 /* XSK RQ and SQ are only created if XDP program is set. */
169 if (!priv->channels.params.xdp_prog)
170 goto remove_pool;
171
172 c = priv->channels.c[ix];
173 mlx5e_rx_res_xsk_deactivate(priv->rx_res, ix);
174 mlx5e_deactivate_xsk(c);
175 mlx5e_close_xsk(c);
176
177 remove_pool:
178 mlx5e_xsk_remove_pool(&priv->xsk, ix);
179 mlx5e_xsk_unmap_pool(priv, pool);
180
181 return 0;
182 }
183
mlx5e_xsk_enable_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)184 static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool,
185 u16 ix)
186 {
187 int err;
188
189 mutex_lock(&priv->state_lock);
190 err = mlx5e_xsk_enable_locked(priv, pool, ix);
191 mutex_unlock(&priv->state_lock);
192
193 return err;
194 }
195
mlx5e_xsk_disable_pool(struct mlx5e_priv * priv,u16 ix)196 static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
197 {
198 int err;
199
200 mutex_lock(&priv->state_lock);
201 err = mlx5e_xsk_disable_locked(priv, ix);
202 mutex_unlock(&priv->state_lock);
203
204 return err;
205 }
206
mlx5e_xsk_setup_pool(struct net_device * dev,struct xsk_buff_pool * pool,u16 qid)207 int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
208 {
209 struct mlx5e_priv *priv = netdev_priv(dev);
210 struct mlx5e_params *params = &priv->channels.params;
211 u16 ix;
212
213 if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
214 return -EINVAL;
215
216 return pool ? mlx5e_xsk_enable_pool(priv, pool, ix) :
217 mlx5e_xsk_disable_pool(priv, ix);
218 }
219