| /linux-6.6.21/drivers/net/ethernet/microchip/lan966x/ |
| D | lan966x_xdp.c | 24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup()
32 xchg(&port->xdp_prog, old_prog); in lan966x_xdp_setup()
77 struct bpf_prog *xdp_prog = port->xdp_prog; in lan966x_xdp_run() local
87 act = bpf_prog_run_xdp(xdp_prog, &xdp); in lan966x_xdp_run()
96 if (xdp_do_redirect(port->dev, &xdp, xdp_prog)) in lan966x_xdp_run()
101 bpf_warn_invalid_xdp_action(port->dev, xdp_prog, act); in lan966x_xdp_run()
104 trace_xdp_exception(port->dev, xdp_prog, act); in lan966x_xdp_run()
|
| /linux-6.6.21/kernel/bpf/ |
| D | devmap.c | 61 struct bpf_prog *xdp_prog; member
68 struct bpf_prog *xdp_prog; member
214 if (dev->xdp_prog) in dev_map_free()
215 bpf_prog_put(dev->xdp_prog); in dev_map_free()
230 if (dev->xdp_prog) in dev_map_free()
231 bpf_prog_put(dev->xdp_prog); in dev_map_free()
322 static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, in dev_map_bpf_prog_run() argument
338 act = bpf_prog_run_xdp(xdp_prog, &xdp); in dev_map_bpf_prog_run()
348 bpf_warn_invalid_xdp_action(NULL, xdp_prog, act); in dev_map_bpf_prog_run()
351 trace_xdp_exception(dev, xdp_prog, act); in dev_map_bpf_prog_run()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 177 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_xdp_attached() local
179 return !!xdp_prog; in bnxt_xdp_attached()
228 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_rx_xdp() local
239 if (!xdp_prog) in bnxt_rx_xdp()
249 act = bpf_prog_run_xdp(xdp_prog, &xdp); in bnxt_rx_xdp()
281 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
308 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
314 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { in bnxt_rx_xdp()
315 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
323 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/sfc/siena/ |
| D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local
263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp()
264 if (!xdp_prog) in efx_do_xdp()
294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp()
320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp()
334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
| /linux-6.6.21/drivers/net/ethernet/sfc/ |
| D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local
263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp()
264 if (!xdp_prog) in efx_do_xdp()
294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp()
320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp()
334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
| /linux-6.6.21/drivers/net/ethernet/freescale/dpaa2/ |
| D | dpaa2-xsk.c | 38 struct bpf_prog *xdp_prog; in dpaa2_xsk_run_xdp() local
44 xdp_prog = READ_ONCE(ch->xdp.prog); in dpaa2_xsk_run_xdp()
45 if (!xdp_prog) in dpaa2_xsk_run_xdp()
59 xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff); in dpaa2_xsk_run_xdp()
66 err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog); in dpaa2_xsk_run_xdp()
85 bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); in dpaa2_xsk_run_xdp()
88 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); in dpaa2_xsk_run_xdp()
122 WARN_ON(priv->xdp_prog); in dpaa2_xsk_rx()
317 if (!priv->xdp_prog) in dpaa2_xsk_wakeup()
|
| /linux-6.6.21/drivers/net/vmxnet3/ |
| D | vmxnet3_xdp.c | 330 struct bpf_prog *xdp_prog; in vmxnet3_process_xdp_small() local
349 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); in vmxnet3_process_xdp_small()
350 if (!xdp_prog) { in vmxnet3_process_xdp_small()
354 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); in vmxnet3_process_xdp_small()
375 struct bpf_prog *xdp_prog; in vmxnet3_process_xdp() local
393 xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog); in vmxnet3_process_xdp()
394 if (!xdp_prog) { in vmxnet3_process_xdp()
398 act = vmxnet3_run_xdp(rq, &xdp, xdp_prog); in vmxnet3_process_xdp()
|
| /linux-6.6.21/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 14 bool xdp_on = READ_ONCE(adapter->xdp_prog); in ixgbe_xsk_pool()
102 struct bpf_prog *xdp_prog; in ixgbe_run_xdp_zc() local
107 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ixgbe_run_xdp_zc()
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
141 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc()
146 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc()
527 if (!READ_ONCE(adapter->xdp_prog)) in ixgbe_xsk_wakeup()
|
| /linux-6.6.21/drivers/net/ethernet/netronome/nfp/nfd3/ |
| D | xsk.c | 121 struct bpf_prog *xdp_prog; in nfp_nfd3_xsk_rx() local
125 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_xsk_rx()
219 act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp); in nfp_nfd3_xsk_rx()
237 if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) { in nfp_nfd3_xsk_rx()
245 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx()
248 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx()
|
| D | dp.c | 553 if (!dp->xdp_prog) { in nfp_nfd3_napi_alloc_one()
568 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfd3_napi_alloc_one()
908 struct bpf_prog *xdp_prog; in nfp_nfd3_rx() local
915 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_rx()
916 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfd3_rx()
1004 if (xdp_prog && !meta.portid) { in nfp_nfd3_rx()
1014 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfd3_rx()
1032 xdp_prog, act); in nfp_nfd3_rx()
1035 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_rx()
1038 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_rx()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/fungible/funeth/ |
| D | funeth_rx.c | 144 struct bpf_prog *xdp_prog; in fun_run_xdp() local
155 xdp_prog = READ_ONCE(q->xdp_prog); in fun_run_xdp()
156 act = bpf_prog_run_xdp(xdp_prog, &xdp); in fun_run_xdp()
177 if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog))) in fun_run_xdp()
183 bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act); in fun_run_xdp()
186 trace_xdp_exception(q->netdev, xdp_prog, act); in fun_run_xdp()
608 WRITE_ONCE(q->xdp_prog, prog); in fun_rxq_set_bpf()
724 q->xdp_prog = fp->xdp_prog; in fun_rxq_create_dev()
725 q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM; in fun_rxq_create_dev()
|
| /linux-6.6.21/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 202 struct bpf_prog *xdp_prog) in i40e_run_xdp_zc() argument
208 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc()
211 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc()
234 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc()
239 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc()
444 struct bpf_prog *xdp_prog; in i40e_clean_rx_irq_zc() local
454 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in i40e_clean_rx_irq_zc()
504 xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog); in i40e_clean_rx_irq_zc()
|
| /linux-6.6.21/drivers/net/ |
| D | veth.c | 63 struct bpf_prog __rcu *xdp_prog; member
584 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) in veth_xdp_flush()
614 struct bpf_prog *xdp_prog; in veth_xdp_rcv_one() local
617 xdp_prog = rcu_dereference(rq->xdp_prog); in veth_xdp_rcv_one()
618 if (likely(xdp_prog)) { in veth_xdp_rcv_one()
627 act = bpf_prog_run_xdp(xdp_prog, xdp); in veth_xdp_rcv_one()
638 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
649 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { in veth_xdp_rcv_one()
658 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
661 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
[all …]
|
| D | tun.c | 208 struct bpf_prog __rcu *xdp_prog; member
829 if (rtnl_dereference(tun->xdp_prog)) in tun_attach()
1204 old_prog = rtnl_dereference(tun->xdp_prog); in tun_xdp_set()
1205 rcu_assign_pointer(tun->xdp_prog, prog); in tun_xdp_set()
1626 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, in tun_xdp_act() argument
1633 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); in tun_xdp_act()
1651 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); in tun_xdp_act()
1654 trace_xdp_exception(tun->dev, xdp_prog, act); in tun_xdp_act()
1671 struct bpf_prog *xdp_prog; in tun_build_skb() local
1679 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_build_skb()
[all …]
|
| D | virtio_net.c | 171 struct bpf_prog __rcu *xdp_prog; member
932 struct bpf_prog *xdp_prog; in virtnet_xdp_xmit() local
946 xdp_prog = rcu_access_pointer(rq->xdp_prog); in virtnet_xdp_xmit()
947 if (!xdp_prog) in virtnet_xdp_xmit()
1017 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, in virtnet_xdp_handler() argument
1026 act = bpf_prog_run_xdp(xdp_prog, xdp); in virtnet_xdp_handler()
1045 trace_xdp_exception(dev, xdp_prog, act); in virtnet_xdp_handler()
1053 err = xdp_do_redirect(dev, xdp, xdp_prog); in virtnet_xdp_handler()
1061 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); in virtnet_xdp_handler()
1064 trace_xdp_exception(dev, xdp_prog, act); in virtnet_xdp_handler()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/aquantia/atlantic/ |
| D | aq_main.c | 137 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_change_mtu()
229 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_fix_features()
231 aq_nic->xdp_prog && features & NETIF_F_LRO) { in aq_ndev_fix_features()
466 need_update = !!aq_nic->xdp_prog != !!prog; in aq_xdp_setup()
470 old_prog = xchg(&aq_nic->xdp_prog, prog); in aq_xdp_setup()
|
| /linux-6.6.21/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| D | params.c | 227 if (params->xdp_prog) in mlx5e_get_linear_rq_headroom()
270 return params->xdp_prog && sz < PAGE_SIZE ? PAGE_SIZE : sz; in mlx5e_rx_get_linear_stride_sz()
424 if (params->xdp_prog) in mlx5e_mpwqe_get_log_stride_size()
717 wqe_bulk_in_bytes = min_t(u32, MAX_WQE_BULK_BYTES(params->xdp_prog), in mlx5e_rx_compute_wqe_bulk_params()
726 split_factor = DIV_ROUND_UP(MAX_WQE_BULK_BYTES(params->xdp_prog), in mlx5e_rx_compute_wqe_bulk_params()
770 params->xdp_prog); in mlx5e_build_rq_frags_info()
771 if (byte_count > max_mtu || params->xdp_prog) { in mlx5e_build_rq_frags_info()
776 params->xdp_prog); in mlx5e_build_rq_frags_info()
796 if (params->xdp_prog) { in mlx5e_build_rq_frags_info()
850 *xdp_frag_size = info->num_frags > 1 && params->xdp_prog ? PAGE_SIZE : 0; in mlx5e_build_rq_frags_info()
[all …]
|
| D | xdp.h | 129 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_enable()
135 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_disable()
|
| /linux-6.6.21/drivers/net/ethernet/intel/ice/ |
| D | ice_xsk.c | 766 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) in ice_run_xdp_zc() argument
771 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp_zc()
774 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ice_run_xdp_zc()
796 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc()
801 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc()
853 struct bpf_prog *xdp_prog; in ice_clean_rx_irq_zc() local
861 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq_zc()
911 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring); in ice_clean_rx_irq_zc()
|
| D | ice_txrx.c | 459 rx_ring->xdp_prog = NULL; in ice_free_rx_ring()
514 WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog); in ice_setup_rx_ring()
560 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring, in ice_run_xdp() argument
566 if (!xdp_prog) in ice_run_xdp()
569 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp()
583 if (xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog)) in ice_run_xdp()
588 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp()
592 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp()
1153 struct bpf_prog *xdp_prog = NULL; in ice_clean_rx_irq() local
1166 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/stmicro/stmmac/ |
| D | stmmac_xdp.c | 123 need_update = !!priv->xdp_prog != !!prog; in stmmac_xdp_set_prog()
127 old_prog = xchg(&priv->xdp_prog, prog); in stmmac_xdp_set_prog()
|
| /linux-6.6.21/drivers/net/ethernet/mellanox/mlx4/ |
| D | en_rx.c | 438 ring->xdp_prog, in mlx4_en_destroy_rx_ring()
719 struct bpf_prog *xdp_prog; in mlx4_en_process_rx_cq() local
732 xdp_prog = rcu_dereference_bh(ring->xdp_prog); in mlx4_en_process_rx_cq()
818 if (xdp_prog) { in mlx4_en_process_rx_cq()
836 act = bpf_prog_run_xdp(xdp_prog, &mxbuf.xdp); in mlx4_en_process_rx_cq()
849 if (likely(!xdp_do_redirect(dev, &mxbuf.xdp, xdp_prog))) { in mlx4_en_process_rx_cq()
856 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
865 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
868 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
871 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
|
| /linux-6.6.21/drivers/net/ethernet/netronome/nfp/nfdk/ |
| D | dp.c | 573 if (!dp->xdp_prog) { in nfp_nfdk_napi_alloc_one()
588 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfdk_napi_alloc_one()
1029 struct bpf_prog *xdp_prog; in nfp_nfdk_rx() local
1037 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfdk_rx()
1038 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfdk_rx()
1123 if (xdp_prog && !meta.portid) { in nfp_nfdk_rx()
1133 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfdk_rx()
1151 xdp_prog, act); in nfp_nfdk_rx()
1154 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfdk_rx()
1157 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfdk_rx()
[all …]
|
| /linux-6.6.21/drivers/net/ethernet/intel/igc/ |
| D | igc_xdp.h | 14 return !!adapter->xdp_prog; in igc_xdp_is_enabled()
|
| /linux-6.6.21/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/ |
| D | pool.c | 116 if (!params->xdp_prog) { in mlx5e_xsk_enable_locked()
176 if (!priv->channels.params.xdp_prog) in mlx5e_xsk_disable_locked()
|