1 /* bnx2x_cmn.c: QLogic Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  * Copyright (c) 2014 QLogic Corporation
5  * All rights reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation.
10  *
11  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
12  * Written by: Eliezer Tamir
13  * Based on code from Michael Chan's bnx2 driver
14  * UDP CSUM errata workaround by Arik Gendelman
15  * Slowpath and fastpath rework by Vladislav Zolotarov
16  * Statistics and Link management by Yitchak Gertner
17  *
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/etherdevice.h>
23 #include <linux/if_vlan.h>
24 #include <linux/interrupt.h>
25 #include <linux/ip.h>
26 #include <linux/crash_dump.h>
27 #include <net/tcp.h>
28 #include <net/gro.h>
29 #include <net/ipv6.h>
30 #include <net/ip6_checksum.h>
31 #include <linux/prefetch.h>
32 #include "bnx2x_cmn.h"
33 #include "bnx2x_init.h"
34 #include "bnx2x_sp.h"
35 
36 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
37 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
38 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
39 static int bnx2x_poll(struct napi_struct *napi, int budget);
40 
bnx2x_add_all_napi_cnic(struct bnx2x * bp)41 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
42 {
43 	int i;
44 
45 	/* Add NAPI objects */
46 	for_each_rx_queue_cnic(bp, i) {
47 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll);
48 	}
49 }
50 
bnx2x_add_all_napi(struct bnx2x * bp)51 static void bnx2x_add_all_napi(struct bnx2x *bp)
52 {
53 	int i;
54 
55 	/* Add NAPI objects */
56 	for_each_eth_queue(bp, i) {
57 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll);
58 	}
59 }
60 
bnx2x_calc_num_queues(struct bnx2x * bp)61 static int bnx2x_calc_num_queues(struct bnx2x *bp)
62 {
63 	int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
64 
65 	/* Reduce memory usage in kdump environment by using only one queue */
66 	if (is_kdump_kernel())
67 		nq = 1;
68 
69 	nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
70 	return nq;
71 }
72 
73 /**
74  * bnx2x_move_fp - move content of the fastpath structure.
75  *
76  * @bp:		driver handle
77  * @from:	source FP index
78  * @to:		destination FP index
79  *
80  * Makes sure the contents of the bp->fp[to].napi is kept
81  * intact. This is done by first copying the napi struct from
82  * the target to the source, and then mem copying the entire
83  * source onto the target. Update txdata pointers and related
84  * content.
85  */
bnx2x_move_fp(struct bnx2x * bp,int from,int to)86 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
87 {
88 	struct bnx2x_fastpath *from_fp = &bp->fp[from];
89 	struct bnx2x_fastpath *to_fp = &bp->fp[to];
90 	struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
91 	struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
92 	struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
93 	struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
94 	int old_max_eth_txqs, new_max_eth_txqs;
95 	int old_txdata_index = 0, new_txdata_index = 0;
96 	struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
97 
98 	/* Copy the NAPI object as it has been already initialized */
99 	from_fp->napi = to_fp->napi;
100 
101 	/* Move bnx2x_fastpath contents */
102 	memcpy(to_fp, from_fp, sizeof(*to_fp));
103 	to_fp->index = to;
104 
105 	/* Retain the tpa_info of the original `to' version as we don't want
106 	 * 2 FPs to contain the same tpa_info pointer.
107 	 */
108 	to_fp->tpa_info = old_tpa_info;
109 
110 	/* move sp_objs contents as well, as their indices match fp ones */
111 	memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
112 
113 	/* move fp_stats contents as well, as their indices match fp ones */
114 	memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
115 
116 	/* Update txdata pointers in fp and move txdata content accordingly:
117 	 * Each fp consumes 'max_cos' txdata structures, so the index should be
118 	 * decremented by max_cos x delta.
119 	 */
120 
121 	old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
122 	new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
123 				(bp)->max_cos;
124 	if (from == FCOE_IDX(bp)) {
125 		old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
126 		new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
127 	}
128 
129 	memcpy(&bp->bnx2x_txq[new_txdata_index],
130 	       &bp->bnx2x_txq[old_txdata_index],
131 	       sizeof(struct bnx2x_fp_txdata));
132 	to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
133 }
134 
135 /**
136  * bnx2x_fill_fw_str - Fill buffer with FW version string.
137  *
138  * @bp:        driver handle
139  * @buf:       character buffer to fill with the fw name
140  * @buf_len:   length of the above buffer
141  *
142  */
bnx2x_fill_fw_str(struct bnx2x * bp,char * buf,size_t buf_len)143 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
144 {
145 	if (IS_PF(bp)) {
146 		u8 phy_fw_ver[PHY_FW_VER_LEN];
147 
148 		phy_fw_ver[0] = '\0';
149 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
150 					     phy_fw_ver, PHY_FW_VER_LEN);
151 		strscpy(buf, bp->fw_ver, buf_len);
152 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
153 			 "bc %d.%d.%d%s%s",
154 			 (bp->common.bc_ver & 0xff0000) >> 16,
155 			 (bp->common.bc_ver & 0xff00) >> 8,
156 			 (bp->common.bc_ver & 0xff),
157 			 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
158 	} else {
159 		bnx2x_vf_fill_fw_str(bp, buf, buf_len);
160 	}
161 }
162 
163 /**
164  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
165  *
166  * @bp:	driver handle
167  * @delta:	number of eth queues which were not allocated
168  */
bnx2x_shrink_eth_fp(struct bnx2x * bp,int delta)169 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
170 {
171 	int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
172 
173 	/* Queue pointer cannot be re-set on an fp-basis, as moving pointer
174 	 * backward along the array could cause memory to be overridden
175 	 */
176 	for (cos = 1; cos < bp->max_cos; cos++) {
177 		for (i = 0; i < old_eth_num - delta; i++) {
178 			struct bnx2x_fastpath *fp = &bp->fp[i];
179 			int new_idx = cos * (old_eth_num - delta) + i;
180 
181 			memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
182 			       sizeof(struct bnx2x_fp_txdata));
183 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
184 		}
185 	}
186 }
187 
188 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
189 
190 /* free skb in the packet ring at pos idx
191  * return idx of last bd freed
192  */
bnx2x_free_tx_pkt(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata,u16 idx,unsigned int * pkts_compl,unsigned int * bytes_compl)193 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
194 			     u16 idx, unsigned int *pkts_compl,
195 			     unsigned int *bytes_compl)
196 {
197 	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
198 	struct eth_tx_start_bd *tx_start_bd;
199 	struct eth_tx_bd *tx_data_bd;
200 	struct sk_buff *skb = tx_buf->skb;
201 	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
202 	int nbd;
203 	u16 split_bd_len = 0;
204 
205 	/* prefetch skb end pointer to speedup dev_kfree_skb() */
206 	prefetch(&skb->end);
207 
208 	DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
209 	   txdata->txq_index, idx, tx_buf, skb);
210 
211 	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
212 
213 	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
214 #ifdef BNX2X_STOP_ON_ERROR
215 	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
216 		BNX2X_ERR("BAD nbd!\n");
217 		bnx2x_panic();
218 	}
219 #endif
220 	new_cons = nbd + tx_buf->first_bd;
221 
222 	/* Get the next bd */
223 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
224 
225 	/* Skip a parse bd... */
226 	--nbd;
227 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
228 
229 	if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
230 		/* Skip second parse bd... */
231 		--nbd;
232 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
233 	}
234 
235 	/* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
236 	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
237 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
238 		split_bd_len = BD_UNMAP_LEN(tx_data_bd);
239 		--nbd;
240 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
241 	}
242 
243 	/* unmap first bd */
244 	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
245 			 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
246 			 DMA_TO_DEVICE);
247 
248 	/* now free frags */
249 	while (nbd > 0) {
250 
251 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
252 		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
253 			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
254 		if (--nbd)
255 			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
256 	}
257 
258 	/* release skb */
259 	WARN_ON(!skb);
260 	if (likely(skb)) {
261 		(*pkts_compl)++;
262 		(*bytes_compl) += skb->len;
263 		dev_kfree_skb_any(skb);
264 	}
265 
266 	tx_buf->first_bd = 0;
267 	tx_buf->skb = NULL;
268 
269 	return new_cons;
270 }
271 
bnx2x_tx_int(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata)272 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
273 {
274 	struct netdev_queue *txq;
275 	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
276 	unsigned int pkts_compl = 0, bytes_compl = 0;
277 
278 #ifdef BNX2X_STOP_ON_ERROR
279 	if (unlikely(bp->panic))
280 		return -1;
281 #endif
282 
283 	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
284 	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
285 	sw_cons = txdata->tx_pkt_cons;
286 
287 	/* Ensure subsequent loads occur after hw_cons */
288 	smp_rmb();
289 
290 	while (sw_cons != hw_cons) {
291 		u16 pkt_cons;
292 
293 		pkt_cons = TX_BD(sw_cons);
294 
295 		DP(NETIF_MSG_TX_DONE,
296 		   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
297 		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
298 
299 		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
300 					    &pkts_compl, &bytes_compl);
301 
302 		sw_cons++;
303 	}
304 
305 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
306 
307 	txdata->tx_pkt_cons = sw_cons;
308 	txdata->tx_bd_cons = bd_cons;
309 
310 	/* Need to make the tx_bd_cons update visible to start_xmit()
311 	 * before checking for netif_tx_queue_stopped().  Without the
312 	 * memory barrier, there is a small possibility that
313 	 * start_xmit() will miss it and cause the queue to be stopped
314 	 * forever.
315 	 * On the other hand we need an rmb() here to ensure the proper
316 	 * ordering of bit testing in the following
317 	 * netif_tx_queue_stopped(txq) call.
318 	 */
319 	smp_mb();
320 
321 	if (unlikely(netif_tx_queue_stopped(txq))) {
322 		/* Taking tx_lock() is needed to prevent re-enabling the queue
323 		 * while it's empty. This could have happen if rx_action() gets
324 		 * suspended in bnx2x_tx_int() after the condition before
325 		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
326 		 *
327 		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
328 		 * sends some packets consuming the whole queue again->
329 		 * stops the queue
330 		 */
331 
332 		__netif_tx_lock(txq, smp_processor_id());
333 
334 		if ((netif_tx_queue_stopped(txq)) &&
335 		    (bp->state == BNX2X_STATE_OPEN) &&
336 		    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
337 			netif_tx_wake_queue(txq);
338 
339 		__netif_tx_unlock(txq);
340 	}
341 	return 0;
342 }
343 
bnx2x_update_last_max_sge(struct bnx2x_fastpath * fp,u16 idx)344 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
345 					     u16 idx)
346 {
347 	u16 last_max = fp->last_max_sge;
348 
349 	if (SUB_S16(idx, last_max) > 0)
350 		fp->last_max_sge = idx;
351 }
352 
bnx2x_update_sge_prod(struct bnx2x_fastpath * fp,u16 sge_len,struct eth_end_agg_rx_cqe * cqe)353 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
354 					 u16 sge_len,
355 					 struct eth_end_agg_rx_cqe *cqe)
356 {
357 	struct bnx2x *bp = fp->bp;
358 	u16 last_max, last_elem, first_elem;
359 	u16 delta = 0;
360 	u16 i;
361 
362 	if (!sge_len)
363 		return;
364 
365 	/* First mark all used pages */
366 	for (i = 0; i < sge_len; i++)
367 		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
368 			RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
369 
370 	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
371 	   sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
372 
373 	/* Here we assume that the last SGE index is the biggest */
374 	prefetch((void *)(fp->sge_mask));
375 	bnx2x_update_last_max_sge(fp,
376 		le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
377 
378 	last_max = RX_SGE(fp->last_max_sge);
379 	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
380 	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
381 
382 	/* If ring is not full */
383 	if (last_elem + 1 != first_elem)
384 		last_elem++;
385 
386 	/* Now update the prod */
387 	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
388 		if (likely(fp->sge_mask[i]))
389 			break;
390 
391 		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
392 		delta += BIT_VEC64_ELEM_SZ;
393 	}
394 
395 	if (delta > 0) {
396 		fp->rx_sge_prod += delta;
397 		/* clear page-end entries */
398 		bnx2x_clear_sge_mask_next_elems(fp);
399 	}
400 
401 	DP(NETIF_MSG_RX_STATUS,
402 	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
403 	   fp->last_max_sge, fp->rx_sge_prod);
404 }
405 
406 /* Get Toeplitz hash value in the skb using the value from the
407  * CQE (calculated by HW).
408  */
bnx2x_get_rxhash(const struct bnx2x * bp,const struct eth_fast_path_rx_cqe * cqe,enum pkt_hash_types * rxhash_type)409 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
410 			    const struct eth_fast_path_rx_cqe *cqe,
411 			    enum pkt_hash_types *rxhash_type)
412 {
413 	/* Get Toeplitz hash from CQE */
414 	if ((bp->dev->features & NETIF_F_RXHASH) &&
415 	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
416 		enum eth_rss_hash_type htype;
417 
418 		htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
419 		*rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
420 				(htype == TCP_IPV6_HASH_TYPE)) ?
421 			       PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
422 
423 		return le32_to_cpu(cqe->rss_hash_result);
424 	}
425 	*rxhash_type = PKT_HASH_TYPE_NONE;
426 	return 0;
427 }
428 
bnx2x_tpa_start(struct bnx2x_fastpath * fp,u16 queue,u16 cons,u16 prod,struct eth_fast_path_rx_cqe * cqe)429 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
430 			    u16 cons, u16 prod,
431 			    struct eth_fast_path_rx_cqe *cqe)
432 {
433 	struct bnx2x *bp = fp->bp;
434 	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
435 	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
436 	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
437 	dma_addr_t mapping;
438 	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
439 	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
440 
441 	/* print error if current state != stop */
442 	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
443 		BNX2X_ERR("start of bin not in stop [%d]\n", queue);
444 
445 	/* Try to map an empty data buffer from the aggregation info  */
446 	mapping = dma_map_single(&bp->pdev->dev,
447 				 first_buf->data + NET_SKB_PAD,
448 				 fp->rx_buf_size, DMA_FROM_DEVICE);
449 	/*
450 	 *  ...if it fails - move the skb from the consumer to the producer
451 	 *  and set the current aggregation state as ERROR to drop it
452 	 *  when TPA_STOP arrives.
453 	 */
454 
455 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
456 		/* Move the BD from the consumer to the producer */
457 		bnx2x_reuse_rx_data(fp, cons, prod);
458 		tpa_info->tpa_state = BNX2X_TPA_ERROR;
459 		return;
460 	}
461 
462 	/* move empty data from pool to prod */
463 	prod_rx_buf->data = first_buf->data;
464 	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
465 	/* point prod_bd to new data */
466 	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
467 	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
468 
469 	/* move partial skb from cons to pool (don't unmap yet) */
470 	*first_buf = *cons_rx_buf;
471 
472 	/* mark bin state as START */
473 	tpa_info->parsing_flags =
474 		le16_to_cpu(cqe->pars_flags.flags);
475 	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
476 	tpa_info->tpa_state = BNX2X_TPA_START;
477 	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
478 	tpa_info->placement_offset = cqe->placement_offset;
479 	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
480 	if (fp->mode == TPA_MODE_GRO) {
481 		u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
482 		tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
483 		tpa_info->gro_size = gro_size;
484 	}
485 
486 #ifdef BNX2X_STOP_ON_ERROR
487 	fp->tpa_queue_used |= (1 << queue);
488 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
489 	   fp->tpa_queue_used);
490 #endif
491 }
492 
493 /* Timestamp option length allowed for TPA aggregation:
494  *
495  *		nop nop kind length echo val
496  */
497 #define TPA_TSTAMP_OPT_LEN	12
498 /**
499  * bnx2x_set_gro_params - compute GRO values
500  *
501  * @skb:		packet skb
502  * @parsing_flags:	parsing flags from the START CQE
503  * @len_on_bd:		total length of the first packet for the
504  *			aggregation.
505  * @pkt_len:		length of all segments
506  * @num_of_coalesced_segs: count of segments
507  *
508  * Approximate value of the MSS for this aggregation calculated using
509  * the first packet of it.
510  * Compute number of aggregated segments, and gso_type.
511  */
bnx2x_set_gro_params(struct sk_buff * skb,u16 parsing_flags,u16 len_on_bd,unsigned int pkt_len,u16 num_of_coalesced_segs)512 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
513 				 u16 len_on_bd, unsigned int pkt_len,
514 				 u16 num_of_coalesced_segs)
515 {
516 	/* TPA aggregation won't have either IP options or TCP options
517 	 * other than timestamp or IPv6 extension headers.
518 	 */
519 	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
520 
521 	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
522 	    PRS_FLAG_OVERETH_IPV6) {
523 		hdrs_len += sizeof(struct ipv6hdr);
524 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
525 	} else {
526 		hdrs_len += sizeof(struct iphdr);
527 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
528 	}
529 
530 	/* Check if there was a TCP timestamp, if there is it's will
531 	 * always be 12 bytes length: nop nop kind length echo val.
532 	 *
533 	 * Otherwise FW would close the aggregation.
534 	 */
535 	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
536 		hdrs_len += TPA_TSTAMP_OPT_LEN;
537 
538 	skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
539 
540 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
541 	 * to skb_shinfo(skb)->gso_segs
542 	 */
543 	NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
544 }
545 
bnx2x_alloc_rx_sge(struct bnx2x * bp,struct bnx2x_fastpath * fp,u16 index,gfp_t gfp_mask)546 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
547 			      u16 index, gfp_t gfp_mask)
548 {
549 	struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
550 	struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
551 	struct bnx2x_alloc_pool *pool = &fp->page_pool;
552 	dma_addr_t mapping;
553 
554 	if (!pool->page) {
555 		pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
556 		if (unlikely(!pool->page))
557 			return -ENOMEM;
558 
559 		pool->offset = 0;
560 	}
561 
562 	mapping = dma_map_page(&bp->pdev->dev, pool->page,
563 			       pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE);
564 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
565 		BNX2X_ERR("Can't map sge\n");
566 		return -ENOMEM;
567 	}
568 
569 	sw_buf->page = pool->page;
570 	sw_buf->offset = pool->offset;
571 
572 	dma_unmap_addr_set(sw_buf, mapping, mapping);
573 
574 	sge->addr_hi = cpu_to_le32(U64_HI(mapping));
575 	sge->addr_lo = cpu_to_le32(U64_LO(mapping));
576 
577 	pool->offset += SGE_PAGE_SIZE;
578 	if (PAGE_SIZE - pool->offset >= SGE_PAGE_SIZE)
579 		get_page(pool->page);
580 	else
581 		pool->page = NULL;
582 	return 0;
583 }
584 
bnx2x_fill_frag_skb(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct bnx2x_agg_info * tpa_info,u16 pages,struct sk_buff * skb,struct eth_end_agg_rx_cqe * cqe,u16 cqe_idx)585 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
586 			       struct bnx2x_agg_info *tpa_info,
587 			       u16 pages,
588 			       struct sk_buff *skb,
589 			       struct eth_end_agg_rx_cqe *cqe,
590 			       u16 cqe_idx)
591 {
592 	struct sw_rx_page *rx_pg, old_rx_pg;
593 	u32 i, frag_len, frag_size;
594 	int err, j, frag_id = 0;
595 	u16 len_on_bd = tpa_info->len_on_bd;
596 	u16 full_page = 0, gro_size = 0;
597 
598 	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
599 
600 	if (fp->mode == TPA_MODE_GRO) {
601 		gro_size = tpa_info->gro_size;
602 		full_page = tpa_info->full_page;
603 	}
604 
605 	/* This is needed in order to enable forwarding support */
606 	if (frag_size)
607 		bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
608 				     le16_to_cpu(cqe->pkt_len),
609 				     le16_to_cpu(cqe->num_of_coalesced_segs));
610 
611 #ifdef BNX2X_STOP_ON_ERROR
612 	if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
613 		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
614 			  pages, cqe_idx);
615 		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
616 		bnx2x_panic();
617 		return -EINVAL;
618 	}
619 #endif
620 
621 	/* Run through the SGL and compose the fragmented skb */
622 	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
623 		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
624 
625 		/* FW gives the indices of the SGE as if the ring is an array
626 		   (meaning that "next" element will consume 2 indices) */
627 		if (fp->mode == TPA_MODE_GRO)
628 			frag_len = min_t(u32, frag_size, (u32)full_page);
629 		else /* LRO */
630 			frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
631 
632 		rx_pg = &fp->rx_page_ring[sge_idx];
633 		old_rx_pg = *rx_pg;
634 
635 		/* If we fail to allocate a substitute page, we simply stop
636 		   where we are and drop the whole packet */
637 		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
638 		if (unlikely(err)) {
639 			bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
640 			return err;
641 		}
642 
643 		dma_unmap_page(&bp->pdev->dev,
644 			       dma_unmap_addr(&old_rx_pg, mapping),
645 			       SGE_PAGE_SIZE, DMA_FROM_DEVICE);
646 		/* Add one frag and update the appropriate fields in the skb */
647 		if (fp->mode == TPA_MODE_LRO)
648 			skb_fill_page_desc(skb, j, old_rx_pg.page,
649 					   old_rx_pg.offset, frag_len);
650 		else { /* GRO */
651 			int rem;
652 			int offset = 0;
653 			for (rem = frag_len; rem > 0; rem -= gro_size) {
654 				int len = rem > gro_size ? gro_size : rem;
655 				skb_fill_page_desc(skb, frag_id++,
656 						   old_rx_pg.page,
657 						   old_rx_pg.offset + offset,
658 						   len);
659 				if (offset)
660 					get_page(old_rx_pg.page);
661 				offset += len;
662 			}
663 		}
664 
665 		skb->data_len += frag_len;
666 		skb->truesize += SGE_PAGES;
667 		skb->len += frag_len;
668 
669 		frag_size -= frag_len;
670 	}
671 
672 	return 0;
673 }
674 
bnx2x_frag_free(const struct bnx2x_fastpath * fp,void * data)675 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
676 {
677 	if (fp->rx_frag_size)
678 		skb_free_frag(data);
679 	else
680 		kfree(data);
681 }
682 
bnx2x_frag_alloc(const struct bnx2x_fastpath * fp,gfp_t gfp_mask)683 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
684 {
685 	if (fp->rx_frag_size) {
686 		/* GFP_KERNEL allocations are used only during initialization */
687 		if (unlikely(gfpflags_allow_blocking(gfp_mask)))
688 			return (void *)__get_free_page(gfp_mask);
689 
690 		return napi_alloc_frag(fp->rx_frag_size);
691 	}
692 
693 	return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
694 }
695 
696 #ifdef CONFIG_INET
bnx2x_gro_ip_csum(struct bnx2x * bp,struct sk_buff * skb)697 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
698 {
699 	const struct iphdr *iph = ip_hdr(skb);
700 	struct tcphdr *th;
701 
702 	skb_set_transport_header(skb, sizeof(struct iphdr));
703 	th = tcp_hdr(skb);
704 
705 	th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
706 				  iph->saddr, iph->daddr, 0);
707 }
708 
bnx2x_gro_ipv6_csum(struct bnx2x * bp,struct sk_buff * skb)709 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
710 {
711 	struct ipv6hdr *iph = ipv6_hdr(skb);
712 	struct tcphdr *th;
713 
714 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
715 	th = tcp_hdr(skb);
716 
717 	th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
718 				  &iph->saddr, &iph->daddr, 0);
719 }
720 
bnx2x_gro_csum(struct bnx2x * bp,struct sk_buff * skb,void (* gro_func)(struct bnx2x *,struct sk_buff *))721 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
722 			    void (*gro_func)(struct bnx2x*, struct sk_buff*))
723 {
724 	skb_reset_network_header(skb);
725 	gro_func(bp, skb);
726 	tcp_gro_complete(skb);
727 }
728 #endif
729 
bnx2x_gro_receive(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct sk_buff * skb)730 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
731 			       struct sk_buff *skb)
732 {
733 #ifdef CONFIG_INET
734 	if (skb_shinfo(skb)->gso_size) {
735 		switch (be16_to_cpu(skb->protocol)) {
736 		case ETH_P_IP:
737 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
738 			break;
739 		case ETH_P_IPV6:
740 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
741 			break;
742 		default:
743 			netdev_WARN_ONCE(bp->dev,
744 					 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
745 					 be16_to_cpu(skb->protocol));
746 		}
747 	}
748 #endif
749 	skb_record_rx_queue(skb, fp->rx_queue);
750 	napi_gro_receive(&fp->napi, skb);
751 }
752 
bnx2x_tpa_stop(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct bnx2x_agg_info * tpa_info,u16 pages,struct eth_end_agg_rx_cqe * cqe,u16 cqe_idx)753 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
754 			   struct bnx2x_agg_info *tpa_info,
755 			   u16 pages,
756 			   struct eth_end_agg_rx_cqe *cqe,
757 			   u16 cqe_idx)
758 {
759 	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
760 	u8 pad = tpa_info->placement_offset;
761 	u16 len = tpa_info->len_on_bd;
762 	struct sk_buff *skb = NULL;
763 	u8 *new_data, *data = rx_buf->data;
764 	u8 old_tpa_state = tpa_info->tpa_state;
765 
766 	tpa_info->tpa_state = BNX2X_TPA_STOP;
767 
768 	/* If we there was an error during the handling of the TPA_START -
769 	 * drop this aggregation.
770 	 */
771 	if (old_tpa_state == BNX2X_TPA_ERROR)
772 		goto drop;
773 
774 	/* Try to allocate the new data */
775 	new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
776 	/* Unmap skb in the pool anyway, as we are going to change
777 	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
778 	   fails. */
779 	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
780 			 fp->rx_buf_size, DMA_FROM_DEVICE);
781 	if (likely(new_data))
782 		skb = build_skb(data, fp->rx_frag_size);
783 
784 	if (likely(skb)) {
785 #ifdef BNX2X_STOP_ON_ERROR
786 		if (pad + len > fp->rx_buf_size) {
787 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
788 				  pad, len, fp->rx_buf_size);
789 			bnx2x_panic();
790 			bnx2x_frag_free(fp, new_data);
791 			return;
792 		}
793 #endif
794 
795 		skb_reserve(skb, pad + NET_SKB_PAD);
796 		skb_put(skb, len);
797 		skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
798 
799 		skb->protocol = eth_type_trans(skb, bp->dev);
800 		skb->ip_summed = CHECKSUM_UNNECESSARY;
801 
802 		if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
803 					 skb, cqe, cqe_idx)) {
804 			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
805 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
806 			bnx2x_gro_receive(bp, fp, skb);
807 		} else {
808 			DP(NETIF_MSG_RX_STATUS,
809 			   "Failed to allocate new pages - dropping packet!\n");
810 			dev_kfree_skb_any(skb);
811 		}
812 
813 		/* put new data in bin */
814 		rx_buf->data = new_data;
815 
816 		return;
817 	}
818 	if (new_data)
819 		bnx2x_frag_free(fp, new_data);
820 drop:
821 	/* drop the packet and keep the buffer in the bin */
822 	DP(NETIF_MSG_RX_STATUS,
823 	   "Failed to allocate or map a new skb - dropping packet!\n");
824 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
825 }
826 
bnx2x_alloc_rx_data(struct bnx2x * bp,struct bnx2x_fastpath * fp,u16 index,gfp_t gfp_mask)827 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
828 			       u16 index, gfp_t gfp_mask)
829 {
830 	u8 *data;
831 	struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
832 	struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
833 	dma_addr_t mapping;
834 
835 	data = bnx2x_frag_alloc(fp, gfp_mask);
836 	if (unlikely(data == NULL))
837 		return -ENOMEM;
838 
839 	mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
840 				 fp->rx_buf_size,
841 				 DMA_FROM_DEVICE);
842 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
843 		bnx2x_frag_free(fp, data);
844 		BNX2X_ERR("Can't map rx data\n");
845 		return -ENOMEM;
846 	}
847 
848 	rx_buf->data = data;
849 	dma_unmap_addr_set(rx_buf, mapping, mapping);
850 
851 	rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
852 	rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
853 
854 	return 0;
855 }
856 
857 static
bnx2x_csum_validate(struct sk_buff * skb,union eth_rx_cqe * cqe,struct bnx2x_fastpath * fp,struct bnx2x_eth_q_stats * qstats)858 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
859 				 struct bnx2x_fastpath *fp,
860 				 struct bnx2x_eth_q_stats *qstats)
861 {
862 	/* Do nothing if no L4 csum validation was done.
863 	 * We do not check whether IP csum was validated. For IPv4 we assume
864 	 * that if the card got as far as validating the L4 csum, it also
865 	 * validated the IP csum. IPv6 has no IP csum.
866 	 */
867 	if (cqe->fast_path_cqe.status_flags &
868 	    ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
869 		return;
870 
871 	/* If L4 validation was done, check if an error was found. */
872 
873 	if (cqe->fast_path_cqe.type_error_flags &
874 	    (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
875 	     ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
876 		qstats->hw_csum_err++;
877 	else
878 		skb->ip_summed = CHECKSUM_UNNECESSARY;
879 }
880 
bnx2x_rx_int(struct bnx2x_fastpath * fp,int budget)881 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
882 {
883 	struct bnx2x *bp = fp->bp;
884 	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
885 	u16 sw_comp_cons, sw_comp_prod;
886 	int rx_pkt = 0;
887 	union eth_rx_cqe *cqe;
888 	struct eth_fast_path_rx_cqe *cqe_fp;
889 
890 #ifdef BNX2X_STOP_ON_ERROR
891 	if (unlikely(bp->panic))
892 		return 0;
893 #endif
894 	if (budget <= 0)
895 		return rx_pkt;
896 
897 	bd_cons = fp->rx_bd_cons;
898 	bd_prod = fp->rx_bd_prod;
899 	bd_prod_fw = bd_prod;
900 	sw_comp_cons = fp->rx_comp_cons;
901 	sw_comp_prod = fp->rx_comp_prod;
902 
903 	comp_ring_cons = RCQ_BD(sw_comp_cons);
904 	cqe = &fp->rx_comp_ring[comp_ring_cons];
905 	cqe_fp = &cqe->fast_path_cqe;
906 
907 	DP(NETIF_MSG_RX_STATUS,
908 	   "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
909 
910 	while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
911 		struct sw_rx_bd *rx_buf = NULL;
912 		struct sk_buff *skb;
913 		u8 cqe_fp_flags;
914 		enum eth_rx_cqe_type cqe_fp_type;
915 		u16 len, pad, queue;
916 		u8 *data;
917 		u32 rxhash;
918 		enum pkt_hash_types rxhash_type;
919 
920 #ifdef BNX2X_STOP_ON_ERROR
921 		if (unlikely(bp->panic))
922 			return 0;
923 #endif
924 
925 		bd_prod = RX_BD(bd_prod);
926 		bd_cons = RX_BD(bd_cons);
927 
928 		/* A rmb() is required to ensure that the CQE is not read
929 		 * before it is written by the adapter DMA.  PCI ordering
930 		 * rules will make sure the other fields are written before
931 		 * the marker at the end of struct eth_fast_path_rx_cqe
932 		 * but without rmb() a weakly ordered processor can process
933 		 * stale data.  Without the barrier TPA state-machine might
934 		 * enter inconsistent state and kernel stack might be
935 		 * provided with incorrect packet description - these lead
936 		 * to various kernel crashed.
937 		 */
938 		rmb();
939 
940 		cqe_fp_flags = cqe_fp->type_error_flags;
941 		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
942 
943 		DP(NETIF_MSG_RX_STATUS,
944 		   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
945 		   CQE_TYPE(cqe_fp_flags),
946 		   cqe_fp_flags, cqe_fp->status_flags,
947 		   le32_to_cpu(cqe_fp->rss_hash_result),
948 		   le16_to_cpu(cqe_fp->vlan_tag),
949 		   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
950 
951 		/* is this a slowpath msg? */
952 		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
953 			bnx2x_sp_event(fp, cqe);
954 			goto next_cqe;
955 		}
956 
957 		rx_buf = &fp->rx_buf_ring[bd_cons];
958 		data = rx_buf->data;
959 
960 		if (!CQE_TYPE_FAST(cqe_fp_type)) {
961 			struct bnx2x_agg_info *tpa_info;
962 			u16 frag_size, pages;
963 #ifdef BNX2X_STOP_ON_ERROR
964 			/* sanity check */
965 			if (fp->mode == TPA_MODE_DISABLED &&
966 			    (CQE_TYPE_START(cqe_fp_type) ||
967 			     CQE_TYPE_STOP(cqe_fp_type)))
968 				BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
969 					  CQE_TYPE(cqe_fp_type));
970 #endif
971 
972 			if (CQE_TYPE_START(cqe_fp_type)) {
973 				u16 queue = cqe_fp->queue_index;
974 				DP(NETIF_MSG_RX_STATUS,
975 				   "calling tpa_start on queue %d\n",
976 				   queue);
977 
978 				bnx2x_tpa_start(fp, queue,
979 						bd_cons, bd_prod,
980 						cqe_fp);
981 
982 				goto next_rx;
983 			}
984 			queue = cqe->end_agg_cqe.queue_index;
985 			tpa_info = &fp->tpa_info[queue];
986 			DP(NETIF_MSG_RX_STATUS,
987 			   "calling tpa_stop on queue %d\n",
988 			   queue);
989 
990 			frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
991 				    tpa_info->len_on_bd;
992 
993 			if (fp->mode == TPA_MODE_GRO)
994 				pages = (frag_size + tpa_info->full_page - 1) /
995 					 tpa_info->full_page;
996 			else
997 				pages = SGE_PAGE_ALIGN(frag_size) >>
998 					SGE_PAGE_SHIFT;
999 
1000 			bnx2x_tpa_stop(bp, fp, tpa_info, pages,
1001 				       &cqe->end_agg_cqe, comp_ring_cons);
1002 #ifdef BNX2X_STOP_ON_ERROR
1003 			if (bp->panic)
1004 				return 0;
1005 #endif
1006 
1007 			bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1008 			goto next_cqe;
1009 		}
1010 		/* non TPA */
1011 		len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1012 		pad = cqe_fp->placement_offset;
1013 		dma_sync_single_for_cpu(&bp->pdev->dev,
1014 					dma_unmap_addr(rx_buf, mapping),
1015 					pad + RX_COPY_THRESH,
1016 					DMA_FROM_DEVICE);
1017 		pad += NET_SKB_PAD;
1018 		prefetch(data + pad); /* speedup eth_type_trans() */
1019 		/* is this an error packet? */
1020 		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1021 			DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1022 			   "ERROR  flags %x  rx packet %u\n",
1023 			   cqe_fp_flags, sw_comp_cons);
1024 			bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1025 			goto reuse_rx;
1026 		}
1027 
1028 		/* Since we don't have a jumbo ring
1029 		 * copy small packets if mtu > 1500
1030 		 */
1031 		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1032 		    (len <= RX_COPY_THRESH)) {
1033 			skb = napi_alloc_skb(&fp->napi, len);
1034 			if (skb == NULL) {
1035 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1036 				   "ERROR  packet dropped because of alloc failure\n");
1037 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1038 				goto reuse_rx;
1039 			}
1040 			memcpy(skb->data, data + pad, len);
1041 			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1042 		} else {
1043 			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1044 						       GFP_ATOMIC) == 0)) {
1045 				dma_unmap_single(&bp->pdev->dev,
1046 						 dma_unmap_addr(rx_buf, mapping),
1047 						 fp->rx_buf_size,
1048 						 DMA_FROM_DEVICE);
1049 				skb = build_skb(data, fp->rx_frag_size);
1050 				if (unlikely(!skb)) {
1051 					bnx2x_frag_free(fp, data);
1052 					bnx2x_fp_qstats(bp, fp)->
1053 							rx_skb_alloc_failed++;
1054 					goto next_rx;
1055 				}
1056 				skb_reserve(skb, pad);
1057 			} else {
1058 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1059 				   "ERROR  packet dropped because of alloc failure\n");
1060 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1061 reuse_rx:
1062 				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1063 				goto next_rx;
1064 			}
1065 		}
1066 
1067 		skb_put(skb, len);
1068 		skb->protocol = eth_type_trans(skb, bp->dev);
1069 
1070 		/* Set Toeplitz hash for a none-LRO skb */
1071 		rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1072 		skb_set_hash(skb, rxhash, rxhash_type);
1073 
1074 		skb_checksum_none_assert(skb);
1075 
1076 		if (bp->dev->features & NETIF_F_RXCSUM)
1077 			bnx2x_csum_validate(skb, cqe, fp,
1078 					    bnx2x_fp_qstats(bp, fp));
1079 
1080 		skb_record_rx_queue(skb, fp->rx_queue);
1081 
1082 		/* Check if this packet was timestamped */
1083 		if (unlikely(cqe->fast_path_cqe.type_error_flags &
1084 			     (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1085 			bnx2x_set_rx_ts(bp, skb);
1086 
1087 		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1088 		    PARSING_FLAGS_VLAN)
1089 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1090 					       le16_to_cpu(cqe_fp->vlan_tag));
1091 
1092 		napi_gro_receive(&fp->napi, skb);
1093 next_rx:
1094 		rx_buf->data = NULL;
1095 
1096 		bd_cons = NEXT_RX_IDX(bd_cons);
1097 		bd_prod = NEXT_RX_IDX(bd_prod);
1098 		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1099 		rx_pkt++;
1100 next_cqe:
1101 		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1102 		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1103 
1104 		/* mark CQE as free */
1105 		BNX2X_SEED_CQE(cqe_fp);
1106 
1107 		if (rx_pkt == budget)
1108 			break;
1109 
1110 		comp_ring_cons = RCQ_BD(sw_comp_cons);
1111 		cqe = &fp->rx_comp_ring[comp_ring_cons];
1112 		cqe_fp = &cqe->fast_path_cqe;
1113 	} /* while */
1114 
1115 	fp->rx_bd_cons = bd_cons;
1116 	fp->rx_bd_prod = bd_prod_fw;
1117 	fp->rx_comp_cons = sw_comp_cons;
1118 	fp->rx_comp_prod = sw_comp_prod;
1119 
1120 	/* Update producers */
1121 	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1122 			     fp->rx_sge_prod);
1123 
1124 	return rx_pkt;
1125 }
1126 
bnx2x_msix_fp_int(int irq,void * fp_cookie)1127 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1128 {
1129 	struct bnx2x_fastpath *fp = fp_cookie;
1130 	struct bnx2x *bp = fp->bp;
1131 	u8 cos;
1132 
1133 	DP(NETIF_MSG_INTR,
1134 	   "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1135 	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
1136 
1137 	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1138 
1139 #ifdef BNX2X_STOP_ON_ERROR
1140 	if (unlikely(bp->panic))
1141 		return IRQ_HANDLED;
1142 #endif
1143 
1144 	/* Handle Rx and Tx according to MSI-X vector */
1145 	for_each_cos_in_tx_queue(fp, cos)
1146 		prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1147 
1148 	prefetch(&fp->sb_running_index[SM_RX_ID]);
1149 	napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1150 
1151 	return IRQ_HANDLED;
1152 }
1153 
1154 /* HW Lock for shared dual port PHYs */
bnx2x_acquire_phy_lock(struct bnx2x * bp)1155 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1156 {
1157 	mutex_lock(&bp->port.phy_mutex);
1158 
1159 	bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1160 }
1161 
bnx2x_release_phy_lock(struct bnx2x * bp)1162 void bnx2x_release_phy_lock(struct bnx2x *bp)
1163 {
1164 	bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1165 
1166 	mutex_unlock(&bp->port.phy_mutex);
1167 }
1168 
1169 /* calculates MF speed according to current linespeed and MF configuration */
bnx2x_get_mf_speed(struct bnx2x * bp)1170 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1171 {
1172 	u16 line_speed = bp->link_vars.line_speed;
1173 	if (IS_MF(bp)) {
1174 		u16 maxCfg = bnx2x_extract_max_cfg(bp,
1175 						   bp->mf_config[BP_VN(bp)]);
1176 
1177 		/* Calculate the current MAX line speed limit for the MF
1178 		 * devices
1179 		 */
1180 		if (IS_MF_PERCENT_BW(bp))
1181 			line_speed = (line_speed * maxCfg) / 100;
1182 		else { /* SD mode */
1183 			u16 vn_max_rate = maxCfg * 100;
1184 
1185 			if (vn_max_rate < line_speed)
1186 				line_speed = vn_max_rate;
1187 		}
1188 	}
1189 
1190 	return line_speed;
1191 }
1192 
1193 /**
1194  * bnx2x_fill_report_data - fill link report data to report
1195  *
1196  * @bp:		driver handle
1197  * @data:	link state to update
1198  *
1199  * It uses a none-atomic bit operations because is called under the mutex.
1200  */
bnx2x_fill_report_data(struct bnx2x * bp,struct bnx2x_link_report_data * data)1201 static void bnx2x_fill_report_data(struct bnx2x *bp,
1202 				   struct bnx2x_link_report_data *data)
1203 {
1204 	memset(data, 0, sizeof(*data));
1205 
1206 	if (IS_PF(bp)) {
1207 		/* Fill the report data: effective line speed */
1208 		data->line_speed = bnx2x_get_mf_speed(bp);
1209 
1210 		/* Link is down */
1211 		if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1212 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1213 				  &data->link_report_flags);
1214 
1215 		if (!BNX2X_NUM_ETH_QUEUES(bp))
1216 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1217 				  &data->link_report_flags);
1218 
1219 		/* Full DUPLEX */
1220 		if (bp->link_vars.duplex == DUPLEX_FULL)
1221 			__set_bit(BNX2X_LINK_REPORT_FD,
1222 				  &data->link_report_flags);
1223 
1224 		/* Rx Flow Control is ON */
1225 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1226 			__set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1227 				  &data->link_report_flags);
1228 
1229 		/* Tx Flow Control is ON */
1230 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1231 			__set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1232 				  &data->link_report_flags);
1233 	} else { /* VF */
1234 		*data = bp->vf_link_vars;
1235 	}
1236 }
1237 
1238 /**
1239  * bnx2x_link_report - report link status to OS.
1240  *
1241  * @bp:		driver handle
1242  *
1243  * Calls the __bnx2x_link_report() under the same locking scheme
1244  * as a link/PHY state managing code to ensure a consistent link
1245  * reporting.
1246  */
1247 
bnx2x_link_report(struct bnx2x * bp)1248 void bnx2x_link_report(struct bnx2x *bp)
1249 {
1250 	bnx2x_acquire_phy_lock(bp);
1251 	__bnx2x_link_report(bp);
1252 	bnx2x_release_phy_lock(bp);
1253 }
1254 
1255 /**
1256  * __bnx2x_link_report - report link status to OS.
1257  *
1258  * @bp:		driver handle
1259  *
1260  * None atomic implementation.
1261  * Should be called under the phy_lock.
1262  */
__bnx2x_link_report(struct bnx2x * bp)1263 void __bnx2x_link_report(struct bnx2x *bp)
1264 {
1265 	struct bnx2x_link_report_data cur_data;
1266 
1267 	if (bp->force_link_down) {
1268 		bp->link_vars.link_up = 0;
1269 		return;
1270 	}
1271 
1272 	/* reread mf_cfg */
1273 	if (IS_PF(bp) && !CHIP_IS_E1(bp))
1274 		bnx2x_read_mf_cfg(bp);
1275 
1276 	/* Read the current link report info */
1277 	bnx2x_fill_report_data(bp, &cur_data);
1278 
1279 	/* Don't report link down or exactly the same link status twice */
1280 	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1281 	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1282 		      &bp->last_reported_link.link_report_flags) &&
1283 	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1284 		      &cur_data.link_report_flags)))
1285 		return;
1286 
1287 	bp->link_cnt++;
1288 
1289 	/* We are going to report a new link parameters now -
1290 	 * remember the current data for the next time.
1291 	 */
1292 	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1293 
1294 	/* propagate status to VFs */
1295 	if (IS_PF(bp))
1296 		bnx2x_iov_link_update(bp);
1297 
1298 	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1299 		     &cur_data.link_report_flags)) {
1300 		netif_carrier_off(bp->dev);
1301 		netdev_err(bp->dev, "NIC Link is Down\n");
1302 		return;
1303 	} else {
1304 		const char *duplex;
1305 		const char *flow;
1306 
1307 		netif_carrier_on(bp->dev);
1308 
1309 		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1310 				       &cur_data.link_report_flags))
1311 			duplex = "full";
1312 		else
1313 			duplex = "half";
1314 
1315 		/* Handle the FC at the end so that only these flags would be
1316 		 * possibly set. This way we may easily check if there is no FC
1317 		 * enabled.
1318 		 */
1319 		if (cur_data.link_report_flags) {
1320 			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1321 				     &cur_data.link_report_flags)) {
1322 				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1323 				     &cur_data.link_report_flags))
1324 					flow = "ON - receive & transmit";
1325 				else
1326 					flow = "ON - receive";
1327 			} else {
1328 				flow = "ON - transmit";
1329 			}
1330 		} else {
1331 			flow = "none";
1332 		}
1333 		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1334 			    cur_data.line_speed, duplex, flow);
1335 	}
1336 }
1337 
bnx2x_set_next_page_sgl(struct bnx2x_fastpath * fp)1338 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1339 {
1340 	int i;
1341 
1342 	for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1343 		struct eth_rx_sge *sge;
1344 
1345 		sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1346 		sge->addr_hi =
1347 			cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1348 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1349 
1350 		sge->addr_lo =
1351 			cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1352 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1353 	}
1354 }
1355 
bnx2x_free_tpa_pool(struct bnx2x * bp,struct bnx2x_fastpath * fp,int last)1356 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1357 				struct bnx2x_fastpath *fp, int last)
1358 {
1359 	int i;
1360 
1361 	for (i = 0; i < last; i++) {
1362 		struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1363 		struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1364 		u8 *data = first_buf->data;
1365 
1366 		if (data == NULL) {
1367 			DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1368 			continue;
1369 		}
1370 		if (tpa_info->tpa_state == BNX2X_TPA_START)
1371 			dma_unmap_single(&bp->pdev->dev,
1372 					 dma_unmap_addr(first_buf, mapping),
1373 					 fp->rx_buf_size, DMA_FROM_DEVICE);
1374 		bnx2x_frag_free(fp, data);
1375 		first_buf->data = NULL;
1376 	}
1377 }
1378 
bnx2x_init_rx_rings_cnic(struct bnx2x * bp)1379 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1380 {
1381 	int j;
1382 
1383 	for_each_rx_queue_cnic(bp, j) {
1384 		struct bnx2x_fastpath *fp = &bp->fp[j];
1385 
1386 		fp->rx_bd_cons = 0;
1387 
1388 		/* Activate BD ring */
1389 		/* Warning!
1390 		 * this will generate an interrupt (to the TSTORM)
1391 		 * must only be done after chip is initialized
1392 		 */
1393 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1394 				     fp->rx_sge_prod);
1395 	}
1396 }
1397 
bnx2x_init_rx_rings(struct bnx2x * bp)1398 void bnx2x_init_rx_rings(struct bnx2x *bp)
1399 {
1400 	int func = BP_FUNC(bp);
1401 	u16 ring_prod;
1402 	int i, j;
1403 
1404 	/* Allocate TPA resources */
1405 	for_each_eth_queue(bp, j) {
1406 		struct bnx2x_fastpath *fp = &bp->fp[j];
1407 
1408 		DP(NETIF_MSG_IFUP,
1409 		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1410 
1411 		if (fp->mode != TPA_MODE_DISABLED) {
1412 			/* Fill the per-aggregation pool */
1413 			for (i = 0; i < MAX_AGG_QS(bp); i++) {
1414 				struct bnx2x_agg_info *tpa_info =
1415 					&fp->tpa_info[i];
1416 				struct sw_rx_bd *first_buf =
1417 					&tpa_info->first_buf;
1418 
1419 				first_buf->data =
1420 					bnx2x_frag_alloc(fp, GFP_KERNEL);
1421 				if (!first_buf->data) {
1422 					BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1423 						  j);
1424 					bnx2x_free_tpa_pool(bp, fp, i);
1425 					fp->mode = TPA_MODE_DISABLED;
1426 					break;
1427 				}
1428 				dma_unmap_addr_set(first_buf, mapping, 0);
1429 				tpa_info->tpa_state = BNX2X_TPA_STOP;
1430 			}
1431 
1432 			/* "next page" elements initialization */
1433 			bnx2x_set_next_page_sgl(fp);
1434 
1435 			/* set SGEs bit mask */
1436 			bnx2x_init_sge_ring_bit_mask(fp);
1437 
1438 			/* Allocate SGEs and initialize the ring elements */
1439 			for (i = 0, ring_prod = 0;
1440 			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1441 
1442 				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1443 						       GFP_KERNEL) < 0) {
1444 					BNX2X_ERR("was only able to allocate %d rx sges\n",
1445 						  i);
1446 					BNX2X_ERR("disabling TPA for queue[%d]\n",
1447 						  j);
1448 					/* Cleanup already allocated elements */
1449 					bnx2x_free_rx_sge_range(bp, fp,
1450 								ring_prod);
1451 					bnx2x_free_tpa_pool(bp, fp,
1452 							    MAX_AGG_QS(bp));
1453 					fp->mode = TPA_MODE_DISABLED;
1454 					ring_prod = 0;
1455 					break;
1456 				}
1457 				ring_prod = NEXT_SGE_IDX(ring_prod);
1458 			}
1459 
1460 			fp->rx_sge_prod = ring_prod;
1461 		}
1462 	}
1463 
1464 	for_each_eth_queue(bp, j) {
1465 		struct bnx2x_fastpath *fp = &bp->fp[j];
1466 
1467 		fp->rx_bd_cons = 0;
1468 
1469 		/* Activate BD ring */
1470 		/* Warning!
1471 		 * this will generate an interrupt (to the TSTORM)
1472 		 * must only be done after chip is initialized
1473 		 */
1474 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1475 				     fp->rx_sge_prod);
1476 
1477 		if (j != 0)
1478 			continue;
1479 
1480 		if (CHIP_IS_E1(bp)) {
1481 			REG_WR(bp, BAR_USTRORM_INTMEM +
1482 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1483 			       U64_LO(fp->rx_comp_mapping));
1484 			REG_WR(bp, BAR_USTRORM_INTMEM +
1485 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1486 			       U64_HI(fp->rx_comp_mapping));
1487 		}
1488 	}
1489 }
1490 
bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath * fp)1491 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1492 {
1493 	u8 cos;
1494 	struct bnx2x *bp = fp->bp;
1495 
1496 	for_each_cos_in_tx_queue(fp, cos) {
1497 		struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1498 		unsigned pkts_compl = 0, bytes_compl = 0;
1499 
1500 		u16 sw_prod = txdata->tx_pkt_prod;
1501 		u16 sw_cons = txdata->tx_pkt_cons;
1502 
1503 		while (sw_cons != sw_prod) {
1504 			bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1505 					  &pkts_compl, &bytes_compl);
1506 			sw_cons++;
1507 		}
1508 
1509 		netdev_tx_reset_queue(
1510 			netdev_get_tx_queue(bp->dev,
1511 					    txdata->txq_index));
1512 	}
1513 }
1514 
bnx2x_free_tx_skbs_cnic(struct bnx2x * bp)1515 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1516 {
1517 	int i;
1518 
1519 	for_each_tx_queue_cnic(bp, i) {
1520 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1521 	}
1522 }
1523 
bnx2x_free_tx_skbs(struct bnx2x * bp)1524 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1525 {
1526 	int i;
1527 
1528 	for_each_eth_queue(bp, i) {
1529 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1530 	}
1531 }
1532 
bnx2x_free_rx_bds(struct bnx2x_fastpath * fp)1533 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1534 {
1535 	struct bnx2x *bp = fp->bp;
1536 	int i;
1537 
1538 	/* ring wasn't allocated */
1539 	if (fp->rx_buf_ring == NULL)
1540 		return;
1541 
1542 	for (i = 0; i < NUM_RX_BD; i++) {
1543 		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1544 		u8 *data = rx_buf->data;
1545 
1546 		if (data == NULL)
1547 			continue;
1548 		dma_unmap_single(&bp->pdev->dev,
1549 				 dma_unmap_addr(rx_buf, mapping),
1550 				 fp->rx_buf_size, DMA_FROM_DEVICE);
1551 
1552 		rx_buf->data = NULL;
1553 		bnx2x_frag_free(fp, data);
1554 	}
1555 }
1556 
bnx2x_free_rx_skbs_cnic(struct bnx2x * bp)1557 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1558 {
1559 	int j;
1560 
1561 	for_each_rx_queue_cnic(bp, j) {
1562 		bnx2x_free_rx_bds(&bp->fp[j]);
1563 	}
1564 }
1565 
bnx2x_free_rx_skbs(struct bnx2x * bp)1566 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1567 {
1568 	int j;
1569 
1570 	for_each_eth_queue(bp, j) {
1571 		struct bnx2x_fastpath *fp = &bp->fp[j];
1572 
1573 		bnx2x_free_rx_bds(fp);
1574 
1575 		if (fp->mode != TPA_MODE_DISABLED)
1576 			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1577 	}
1578 }
1579 
bnx2x_free_skbs_cnic(struct bnx2x * bp)1580 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1581 {
1582 	bnx2x_free_tx_skbs_cnic(bp);
1583 	bnx2x_free_rx_skbs_cnic(bp);
1584 }
1585 
bnx2x_free_skbs(struct bnx2x * bp)1586 void bnx2x_free_skbs(struct bnx2x *bp)
1587 {
1588 	bnx2x_free_tx_skbs(bp);
1589 	bnx2x_free_rx_skbs(bp);
1590 }
1591 
bnx2x_update_max_mf_config(struct bnx2x * bp,u32 value)1592 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1593 {
1594 	/* load old values */
1595 	u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1596 
1597 	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1598 		/* leave all but MAX value */
1599 		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1600 
1601 		/* set new MAX value */
1602 		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1603 				& FUNC_MF_CFG_MAX_BW_MASK;
1604 
1605 		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1606 	}
1607 }
1608 
1609 /**
1610  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1611  *
1612  * @bp:		driver handle
1613  * @nvecs:	number of vectors to be released
1614  */
bnx2x_free_msix_irqs(struct bnx2x * bp,int nvecs)1615 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1616 {
1617 	int i, offset = 0;
1618 
1619 	if (nvecs == offset)
1620 		return;
1621 
1622 	/* VFs don't have a default SB */
1623 	if (IS_PF(bp)) {
1624 		free_irq(bp->msix_table[offset].vector, bp->dev);
1625 		DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1626 		   bp->msix_table[offset].vector);
1627 		offset++;
1628 	}
1629 
1630 	if (CNIC_SUPPORT(bp)) {
1631 		if (nvecs == offset)
1632 			return;
1633 		offset++;
1634 	}
1635 
1636 	for_each_eth_queue(bp, i) {
1637 		if (nvecs == offset)
1638 			return;
1639 		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1640 		   i, bp->msix_table[offset].vector);
1641 
1642 		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1643 	}
1644 }
1645 
bnx2x_free_irq(struct bnx2x * bp)1646 void bnx2x_free_irq(struct bnx2x *bp)
1647 {
1648 	if (bp->flags & USING_MSIX_FLAG &&
1649 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1650 		int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1651 
1652 		/* vfs don't have a default status block */
1653 		if (IS_PF(bp))
1654 			nvecs++;
1655 
1656 		bnx2x_free_msix_irqs(bp, nvecs);
1657 	} else {
1658 		free_irq(bp->dev->irq, bp->dev);
1659 	}
1660 }
1661 
bnx2x_enable_msix(struct bnx2x * bp)1662 int bnx2x_enable_msix(struct bnx2x *bp)
1663 {
1664 	int msix_vec = 0, i, rc;
1665 
1666 	/* VFs don't have a default status block */
1667 	if (IS_PF(bp)) {
1668 		bp->msix_table[msix_vec].entry = msix_vec;
1669 		BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1670 			       bp->msix_table[0].entry);
1671 		msix_vec++;
1672 	}
1673 
1674 	/* Cnic requires an msix vector for itself */
1675 	if (CNIC_SUPPORT(bp)) {
1676 		bp->msix_table[msix_vec].entry = msix_vec;
1677 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1678 			       msix_vec, bp->msix_table[msix_vec].entry);
1679 		msix_vec++;
1680 	}
1681 
1682 	/* We need separate vectors for ETH queues only (not FCoE) */
1683 	for_each_eth_queue(bp, i) {
1684 		bp->msix_table[msix_vec].entry = msix_vec;
1685 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1686 			       msix_vec, msix_vec, i);
1687 		msix_vec++;
1688 	}
1689 
1690 	DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1691 	   msix_vec);
1692 
1693 	rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1694 				   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1695 	/*
1696 	 * reconfigure number of tx/rx queues according to available
1697 	 * MSI-X vectors
1698 	 */
1699 	if (rc == -ENOSPC) {
1700 		/* Get by with single vector */
1701 		rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1702 		if (rc < 0) {
1703 			BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1704 				       rc);
1705 			goto no_msix;
1706 		}
1707 
1708 		BNX2X_DEV_INFO("Using single MSI-X vector\n");
1709 		bp->flags |= USING_SINGLE_MSIX_FLAG;
1710 
1711 		BNX2X_DEV_INFO("set number of queues to 1\n");
1712 		bp->num_ethernet_queues = 1;
1713 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1714 	} else if (rc < 0) {
1715 		BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1716 		goto no_msix;
1717 	} else if (rc < msix_vec) {
1718 		/* how less vectors we will have? */
1719 		int diff = msix_vec - rc;
1720 
1721 		BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1722 
1723 		/*
1724 		 * decrease number of queues by number of unallocated entries
1725 		 */
1726 		bp->num_ethernet_queues -= diff;
1727 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1728 
1729 		BNX2X_DEV_INFO("New queue configuration set: %d\n",
1730 			       bp->num_queues);
1731 	}
1732 
1733 	bp->flags |= USING_MSIX_FLAG;
1734 
1735 	return 0;
1736 
1737 no_msix:
1738 	/* fall to INTx if not enough memory */
1739 	if (rc == -ENOMEM)
1740 		bp->flags |= DISABLE_MSI_FLAG;
1741 
1742 	return rc;
1743 }
1744 
bnx2x_req_msix_irqs(struct bnx2x * bp)1745 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1746 {
1747 	int i, rc, offset = 0;
1748 
1749 	/* no default status block for vf */
1750 	if (IS_PF(bp)) {
1751 		rc = request_irq(bp->msix_table[offset++].vector,
1752 				 bnx2x_msix_sp_int, 0,
1753 				 bp->dev->name, bp->dev);
1754 		if (rc) {
1755 			BNX2X_ERR("request sp irq failed\n");
1756 			return -EBUSY;
1757 		}
1758 	}
1759 
1760 	if (CNIC_SUPPORT(bp))
1761 		offset++;
1762 
1763 	for_each_eth_queue(bp, i) {
1764 		struct bnx2x_fastpath *fp = &bp->fp[i];
1765 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1766 			 bp->dev->name, i);
1767 
1768 		rc = request_irq(bp->msix_table[offset].vector,
1769 				 bnx2x_msix_fp_int, 0, fp->name, fp);
1770 		if (rc) {
1771 			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1772 			      bp->msix_table[offset].vector, rc);
1773 			bnx2x_free_msix_irqs(bp, offset);
1774 			return -EBUSY;
1775 		}
1776 
1777 		offset++;
1778 	}
1779 
1780 	i = BNX2X_NUM_ETH_QUEUES(bp);
1781 	if (IS_PF(bp)) {
1782 		offset = 1 + CNIC_SUPPORT(bp);
1783 		netdev_info(bp->dev,
1784 			    "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1785 			    bp->msix_table[0].vector,
1786 			    0, bp->msix_table[offset].vector,
1787 			    i - 1, bp->msix_table[offset + i - 1].vector);
1788 	} else {
1789 		offset = CNIC_SUPPORT(bp);
1790 		netdev_info(bp->dev,
1791 			    "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1792 			    0, bp->msix_table[offset].vector,
1793 			    i - 1, bp->msix_table[offset + i - 1].vector);
1794 	}
1795 	return 0;
1796 }
1797 
bnx2x_enable_msi(struct bnx2x * bp)1798 int bnx2x_enable_msi(struct bnx2x *bp)
1799 {
1800 	int rc;
1801 
1802 	rc = pci_enable_msi(bp->pdev);
1803 	if (rc) {
1804 		BNX2X_DEV_INFO("MSI is not attainable\n");
1805 		return -1;
1806 	}
1807 	bp->flags |= USING_MSI_FLAG;
1808 
1809 	return 0;
1810 }
1811 
bnx2x_req_irq(struct bnx2x * bp)1812 static int bnx2x_req_irq(struct bnx2x *bp)
1813 {
1814 	unsigned long flags;
1815 	unsigned int irq;
1816 
1817 	if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1818 		flags = 0;
1819 	else
1820 		flags = IRQF_SHARED;
1821 
1822 	if (bp->flags & USING_MSIX_FLAG)
1823 		irq = bp->msix_table[0].vector;
1824 	else
1825 		irq = bp->pdev->irq;
1826 
1827 	return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1828 }
1829 
bnx2x_setup_irqs(struct bnx2x * bp)1830 static int bnx2x_setup_irqs(struct bnx2x *bp)
1831 {
1832 	int rc = 0;
1833 	if (bp->flags & USING_MSIX_FLAG &&
1834 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1835 		rc = bnx2x_req_msix_irqs(bp);
1836 		if (rc)
1837 			return rc;
1838 	} else {
1839 		rc = bnx2x_req_irq(bp);
1840 		if (rc) {
1841 			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1842 			return rc;
1843 		}
1844 		if (bp->flags & USING_MSI_FLAG) {
1845 			bp->dev->irq = bp->pdev->irq;
1846 			netdev_info(bp->dev, "using MSI IRQ %d\n",
1847 				    bp->dev->irq);
1848 		}
1849 		if (bp->flags & USING_MSIX_FLAG) {
1850 			bp->dev->irq = bp->msix_table[0].vector;
1851 			netdev_info(bp->dev, "using MSIX IRQ %d\n",
1852 				    bp->dev->irq);
1853 		}
1854 	}
1855 
1856 	return 0;
1857 }
1858 
bnx2x_napi_enable_cnic(struct bnx2x * bp)1859 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1860 {
1861 	int i;
1862 
1863 	for_each_rx_queue_cnic(bp, i) {
1864 		napi_enable(&bnx2x_fp(bp, i, napi));
1865 	}
1866 }
1867 
bnx2x_napi_enable(struct bnx2x * bp)1868 static void bnx2x_napi_enable(struct bnx2x *bp)
1869 {
1870 	int i;
1871 
1872 	for_each_eth_queue(bp, i) {
1873 		napi_enable(&bnx2x_fp(bp, i, napi));
1874 	}
1875 }
1876 
bnx2x_napi_disable_cnic(struct bnx2x * bp)1877 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1878 {
1879 	int i;
1880 
1881 	for_each_rx_queue_cnic(bp, i) {
1882 		napi_disable(&bnx2x_fp(bp, i, napi));
1883 	}
1884 }
1885 
bnx2x_napi_disable(struct bnx2x * bp)1886 static void bnx2x_napi_disable(struct bnx2x *bp)
1887 {
1888 	int i;
1889 
1890 	for_each_eth_queue(bp, i) {
1891 		napi_disable(&bnx2x_fp(bp, i, napi));
1892 	}
1893 }
1894 
bnx2x_netif_start(struct bnx2x * bp)1895 void bnx2x_netif_start(struct bnx2x *bp)
1896 {
1897 	if (netif_running(bp->dev)) {
1898 		bnx2x_napi_enable(bp);
1899 		if (CNIC_LOADED(bp))
1900 			bnx2x_napi_enable_cnic(bp);
1901 		bnx2x_int_enable(bp);
1902 		if (bp->state == BNX2X_STATE_OPEN)
1903 			netif_tx_wake_all_queues(bp->dev);
1904 	}
1905 }
1906 
bnx2x_netif_stop(struct bnx2x * bp,int disable_hw)1907 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1908 {
1909 	bnx2x_int_disable_sync(bp, disable_hw);
1910 	bnx2x_napi_disable(bp);
1911 	if (CNIC_LOADED(bp))
1912 		bnx2x_napi_disable_cnic(bp);
1913 }
1914 
bnx2x_select_queue(struct net_device * dev,struct sk_buff * skb,struct net_device * sb_dev)1915 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1916 		       struct net_device *sb_dev)
1917 {
1918 	struct bnx2x *bp = netdev_priv(dev);
1919 
1920 	if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1921 		struct ethhdr *hdr = (struct ethhdr *)skb->data;
1922 		u16 ether_type = ntohs(hdr->h_proto);
1923 
1924 		/* Skip VLAN tag if present */
1925 		if (ether_type == ETH_P_8021Q) {
1926 			struct vlan_ethhdr *vhdr =
1927 				(struct vlan_ethhdr *)skb->data;
1928 
1929 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1930 		}
1931 
1932 		/* If ethertype is FCoE or FIP - use FCoE ring */
1933 		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1934 			return bnx2x_fcoe_tx(bp, txq_index);
1935 	}
1936 
1937 	/* select a non-FCoE queue */
1938 	return netdev_pick_tx(dev, skb, NULL) %
1939 			(BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos);
1940 }
1941 
bnx2x_set_num_queues(struct bnx2x * bp)1942 void bnx2x_set_num_queues(struct bnx2x *bp)
1943 {
1944 	/* RSS queues */
1945 	bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1946 
1947 	/* override in STORAGE SD modes */
1948 	if (IS_MF_STORAGE_ONLY(bp))
1949 		bp->num_ethernet_queues = 1;
1950 
1951 	/* Add special queues */
1952 	bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1953 	bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1954 
1955 	BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1956 }
1957 
1958 /**
1959  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1960  *
1961  * @bp:		Driver handle
1962  * @include_cnic: handle cnic case
1963  *
1964  * We currently support for at most 16 Tx queues for each CoS thus we will
1965  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1966  * bp->max_cos.
1967  *
1968  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1969  * index after all ETH L2 indices.
1970  *
1971  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1972  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1973  * 16..31,...) with indices that are not coupled with any real Tx queue.
1974  *
1975  * The proper configuration of skb->queue_mapping is handled by
1976  * bnx2x_select_queue() and __skb_tx_hash().
1977  *
1978  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1979  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1980  */
bnx2x_set_real_num_queues(struct bnx2x * bp,int include_cnic)1981 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1982 {
1983 	int rc, tx, rx;
1984 
1985 	tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1986 	rx = BNX2X_NUM_ETH_QUEUES(bp);
1987 
1988 /* account for fcoe queue */
1989 	if (include_cnic && !NO_FCOE(bp)) {
1990 		rx++;
1991 		tx++;
1992 	}
1993 
1994 	rc = netif_set_real_num_tx_queues(bp->dev, tx);
1995 	if (rc) {
1996 		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1997 		return rc;
1998 	}
1999 	rc = netif_set_real_num_rx_queues(bp->dev, rx);
2000 	if (rc) {
2001 		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2002 		return rc;
2003 	}
2004 
2005 	DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2006 			  tx, rx);
2007 
2008 	return rc;
2009 }
2010 
bnx2x_set_rx_buf_size(struct bnx2x * bp)2011 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2012 {
2013 	int i;
2014 
2015 	for_each_queue(bp, i) {
2016 		struct bnx2x_fastpath *fp = &bp->fp[i];
2017 		u32 mtu;
2018 
2019 		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
2020 		if (IS_FCOE_IDX(i))
2021 			/*
2022 			 * Although there are no IP frames expected to arrive to
2023 			 * this ring we still want to add an
2024 			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2025 			 * overrun attack.
2026 			 */
2027 			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2028 		else
2029 			mtu = bp->dev->mtu;
2030 		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2031 				  IP_HEADER_ALIGNMENT_PADDING +
2032 				  ETH_OVERHEAD +
2033 				  mtu +
2034 				  BNX2X_FW_RX_ALIGN_END;
2035 		fp->rx_buf_size = SKB_DATA_ALIGN(fp->rx_buf_size);
2036 		/* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2037 		if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2038 			fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2039 		else
2040 			fp->rx_frag_size = 0;
2041 	}
2042 }
2043 
bnx2x_init_rss(struct bnx2x * bp)2044 static int bnx2x_init_rss(struct bnx2x *bp)
2045 {
2046 	int i;
2047 	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2048 
2049 	/* Prepare the initial contents for the indirection table if RSS is
2050 	 * enabled
2051 	 */
2052 	for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2053 		bp->rss_conf_obj.ind_table[i] =
2054 			bp->fp->cl_id +
2055 			ethtool_rxfh_indir_default(i, num_eth_queues);
2056 
2057 	/*
2058 	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2059 	 * per-port, so if explicit configuration is needed , do it only
2060 	 * for a PMF.
2061 	 *
2062 	 * For 57712 and newer on the other hand it's a per-function
2063 	 * configuration.
2064 	 */
2065 	return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2066 }
2067 
bnx2x_rss(struct bnx2x * bp,struct bnx2x_rss_config_obj * rss_obj,bool config_hash,bool enable)2068 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2069 	      bool config_hash, bool enable)
2070 {
2071 	struct bnx2x_config_rss_params params = {NULL};
2072 
2073 	/* Although RSS is meaningless when there is a single HW queue we
2074 	 * still need it enabled in order to have HW Rx hash generated.
2075 	 *
2076 	 * if (!is_eth_multi(bp))
2077 	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2078 	 */
2079 
2080 	params.rss_obj = rss_obj;
2081 
2082 	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2083 
2084 	if (enable) {
2085 		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2086 
2087 		/* RSS configuration */
2088 		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2089 		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2090 		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2091 		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2092 		if (rss_obj->udp_rss_v4)
2093 			__set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2094 		if (rss_obj->udp_rss_v6)
2095 			__set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2096 
2097 		if (!CHIP_IS_E1x(bp)) {
2098 			/* valid only for TUNN_MODE_VXLAN tunnel mode */
2099 			__set_bit(BNX2X_RSS_IPV4_VXLAN, &params.rss_flags);
2100 			__set_bit(BNX2X_RSS_IPV6_VXLAN, &params.rss_flags);
2101 
2102 			/* valid only for TUNN_MODE_GRE tunnel mode */
2103 			__set_bit(BNX2X_RSS_TUNN_INNER_HDRS, &params.rss_flags);
2104 		}
2105 	} else {
2106 		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2107 	}
2108 
2109 	/* Hash bits */
2110 	params.rss_result_mask = MULTI_MASK;
2111 
2112 	memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2113 
2114 	if (config_hash) {
2115 		/* RSS keys */
2116 		netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2117 		__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2118 	}
2119 
2120 	if (IS_PF(bp))
2121 		return bnx2x_config_rss(bp, &params);
2122 	else
2123 		return bnx2x_vfpf_config_rss(bp, &params);
2124 }
2125 
bnx2x_init_hw(struct bnx2x * bp,u32 load_code)2126 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2127 {
2128 	struct bnx2x_func_state_params func_params = {NULL};
2129 
2130 	/* Prepare parameters for function state transitions */
2131 	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2132 
2133 	func_params.f_obj = &bp->func_obj;
2134 	func_params.cmd = BNX2X_F_CMD_HW_INIT;
2135 
2136 	func_params.params.hw_init.load_phase = load_code;
2137 
2138 	return bnx2x_func_state_change(bp, &func_params);
2139 }
2140 
2141 /*
2142  * Cleans the object that have internal lists without sending
2143  * ramrods. Should be run when interrupts are disabled.
2144  */
bnx2x_squeeze_objects(struct bnx2x * bp)2145 void bnx2x_squeeze_objects(struct bnx2x *bp)
2146 {
2147 	int rc;
2148 	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2149 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
2150 	struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2151 
2152 	/***************** Cleanup MACs' object first *************************/
2153 
2154 	/* Wait for completion of requested */
2155 	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2156 	/* Perform a dry cleanup */
2157 	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2158 
2159 	/* Clean ETH primary MAC */
2160 	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2161 	rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2162 				 &ramrod_flags);
2163 	if (rc != 0)
2164 		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2165 
2166 	/* Cleanup UC list */
2167 	vlan_mac_flags = 0;
2168 	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2169 	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2170 				 &ramrod_flags);
2171 	if (rc != 0)
2172 		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2173 
2174 	/***************** Now clean mcast object *****************************/
2175 	rparam.mcast_obj = &bp->mcast_obj;
2176 	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2177 
2178 	/* Add a DEL command... - Since we're doing a driver cleanup only,
2179 	 * we take a lock surrounding both the initial send and the CONTs,
2180 	 * as we don't want a true completion to disrupt us in the middle.
2181 	 */
2182 	netif_addr_lock_bh(bp->dev);
2183 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2184 	if (rc < 0)
2185 		BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2186 			  rc);
2187 
2188 	/* ...and wait until all pending commands are cleared */
2189 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2190 	while (rc != 0) {
2191 		if (rc < 0) {
2192 			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2193 				  rc);
2194 			netif_addr_unlock_bh(bp->dev);
2195 			return;
2196 		}
2197 
2198 		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2199 	}
2200 	netif_addr_unlock_bh(bp->dev);
2201 }
2202 
2203 #ifndef BNX2X_STOP_ON_ERROR
2204 #define LOAD_ERROR_EXIT(bp, label) \
2205 	do { \
2206 		(bp)->state = BNX2X_STATE_ERROR; \
2207 		goto label; \
2208 	} while (0)
2209 
2210 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2211 	do { \
2212 		bp->cnic_loaded = false; \
2213 		goto label; \
2214 	} while (0)
2215 #else /*BNX2X_STOP_ON_ERROR*/
2216 #define LOAD_ERROR_EXIT(bp, label) \
2217 	do { \
2218 		(bp)->state = BNX2X_STATE_ERROR; \
2219 		(bp)->panic = 1; \
2220 		return -EBUSY; \
2221 	} while (0)
2222 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2223 	do { \
2224 		bp->cnic_loaded = false; \
2225 		(bp)->panic = 1; \
2226 		return -EBUSY; \
2227 	} while (0)
2228 #endif /*BNX2X_STOP_ON_ERROR*/
2229 
bnx2x_free_fw_stats_mem(struct bnx2x * bp)2230 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2231 {
2232 	BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2233 		       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2234 	return;
2235 }
2236 
bnx2x_alloc_fw_stats_mem(struct bnx2x * bp)2237 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2238 {
2239 	int num_groups, vf_headroom = 0;
2240 	int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2241 
2242 	/* number of queues for statistics is number of eth queues + FCoE */
2243 	u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2244 
2245 	/* Total number of FW statistics requests =
2246 	 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2247 	 * and fcoe l2 queue) stats + num of queues (which includes another 1
2248 	 * for fcoe l2 queue if applicable)
2249 	 */
2250 	bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2251 
2252 	/* vf stats appear in the request list, but their data is allocated by
2253 	 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2254 	 * it is used to determine where to place the vf stats queries in the
2255 	 * request struct
2256 	 */
2257 	if (IS_SRIOV(bp))
2258 		vf_headroom = bnx2x_vf_headroom(bp);
2259 
2260 	/* Request is built from stats_query_header and an array of
2261 	 * stats_query_cmd_group each of which contains
2262 	 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2263 	 * configured in the stats_query_header.
2264 	 */
2265 	num_groups =
2266 		(((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2267 		 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2268 		 1 : 0));
2269 
2270 	DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2271 	   bp->fw_stats_num, vf_headroom, num_groups);
2272 	bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2273 		num_groups * sizeof(struct stats_query_cmd_group);
2274 
2275 	/* Data for statistics requests + stats_counter
2276 	 * stats_counter holds per-STORM counters that are incremented
2277 	 * when STORM has finished with the current request.
2278 	 * memory for FCoE offloaded statistics are counted anyway,
2279 	 * even if they will not be sent.
2280 	 * VF stats are not accounted for here as the data of VF stats is stored
2281 	 * in memory allocated by the VF, not here.
2282 	 */
2283 	bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2284 		sizeof(struct per_pf_stats) +
2285 		sizeof(struct fcoe_statistics_params) +
2286 		sizeof(struct per_queue_stats) * num_queue_stats +
2287 		sizeof(struct stats_counter);
2288 
2289 	bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2290 				       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2291 	if (!bp->fw_stats)
2292 		goto alloc_mem_err;
2293 
2294 	/* Set shortcuts */
2295 	bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2296 	bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2297 	bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2298 		((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2299 	bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2300 		bp->fw_stats_req_sz;
2301 
2302 	DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2303 	   U64_HI(bp->fw_stats_req_mapping),
2304 	   U64_LO(bp->fw_stats_req_mapping));
2305 	DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2306 	   U64_HI(bp->fw_stats_data_mapping),
2307 	   U64_LO(bp->fw_stats_data_mapping));
2308 	return 0;
2309 
2310 alloc_mem_err:
2311 	bnx2x_free_fw_stats_mem(bp);
2312 	BNX2X_ERR("Can't allocate FW stats memory\n");
2313 	return -ENOMEM;
2314 }
2315 
2316 /* send load request to mcp and analyze response */
bnx2x_nic_load_request(struct bnx2x * bp,u32 * load_code)2317 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2318 {
2319 	u32 param;
2320 
2321 	/* init fw_seq */
2322 	bp->fw_seq =
2323 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2324 		 DRV_MSG_SEQ_NUMBER_MASK);
2325 	BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2326 
2327 	/* Get current FW pulse sequence */
2328 	bp->fw_drv_pulse_wr_seq =
2329 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2330 		 DRV_PULSE_SEQ_MASK);
2331 	BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2332 
2333 	param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2334 
2335 	if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2336 		param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2337 
2338 	/* load request */
2339 	(*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2340 
2341 	/* if mcp fails to respond we must abort */
2342 	if (!(*load_code)) {
2343 		BNX2X_ERR("MCP response failure, aborting\n");
2344 		return -EBUSY;
2345 	}
2346 
2347 	/* If mcp refused (e.g. other port is in diagnostic mode) we
2348 	 * must abort
2349 	 */
2350 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2351 		BNX2X_ERR("MCP refused load request, aborting\n");
2352 		return -EBUSY;
2353 	}
2354 	return 0;
2355 }
2356 
2357 /* check whether another PF has already loaded FW to chip. In
2358  * virtualized environments a pf from another VM may have already
2359  * initialized the device including loading FW
2360  */
bnx2x_compare_fw_ver(struct bnx2x * bp,u32 load_code,bool print_err)2361 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2362 {
2363 	/* is another pf loaded on this engine? */
2364 	if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2365 	    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2366 		u8 loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng;
2367 		u32 loaded_fw;
2368 
2369 		/* read loaded FW from chip */
2370 		loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2371 
2372 		loaded_fw_major = loaded_fw & 0xff;
2373 		loaded_fw_minor = (loaded_fw >> 8) & 0xff;
2374 		loaded_fw_rev = (loaded_fw >> 16) & 0xff;
2375 		loaded_fw_eng = (loaded_fw >> 24) & 0xff;
2376 
2377 		DP(BNX2X_MSG_SP, "loaded fw 0x%x major 0x%x minor 0x%x rev 0x%x eng 0x%x\n",
2378 		   loaded_fw, loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng);
2379 
2380 		/* abort nic load if version mismatch */
2381 		if (loaded_fw_major != BCM_5710_FW_MAJOR_VERSION ||
2382 		    loaded_fw_minor != BCM_5710_FW_MINOR_VERSION ||
2383 		    loaded_fw_eng != BCM_5710_FW_ENGINEERING_VERSION ||
2384 		    loaded_fw_rev < BCM_5710_FW_REVISION_VERSION_V15) {
2385 			if (print_err)
2386 				BNX2X_ERR("loaded FW incompatible. Aborting\n");
2387 			else
2388 				BNX2X_DEV_INFO("loaded FW incompatible, possibly due to MF UNDI\n");
2389 
2390 			return -EBUSY;
2391 		}
2392 	}
2393 	return 0;
2394 }
2395 
2396 /* returns the "mcp load_code" according to global load_count array */
bnx2x_nic_load_no_mcp(struct bnx2x * bp,int port)2397 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2398 {
2399 	int path = BP_PATH(bp);
2400 
2401 	DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2402 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2403 	   bnx2x_load_count[path][2]);
2404 	bnx2x_load_count[path][0]++;
2405 	bnx2x_load_count[path][1 + port]++;
2406 	DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2407 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2408 	   bnx2x_load_count[path][2]);
2409 	if (bnx2x_load_count[path][0] == 1)
2410 		return FW_MSG_CODE_DRV_LOAD_COMMON;
2411 	else if (bnx2x_load_count[path][1 + port] == 1)
2412 		return FW_MSG_CODE_DRV_LOAD_PORT;
2413 	else
2414 		return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2415 }
2416 
2417 /* mark PMF if applicable */
bnx2x_nic_load_pmf(struct bnx2x * bp,u32 load_code)2418 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2419 {
2420 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2421 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2422 	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2423 		bp->port.pmf = 1;
2424 		/* We need the barrier to ensure the ordering between the
2425 		 * writing to bp->port.pmf here and reading it from the
2426 		 * bnx2x_periodic_task().
2427 		 */
2428 		smp_mb();
2429 	} else {
2430 		bp->port.pmf = 0;
2431 	}
2432 
2433 	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2434 }
2435 
bnx2x_nic_load_afex_dcc(struct bnx2x * bp,int load_code)2436 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2437 {
2438 	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2439 	     (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2440 	    (bp->common.shmem2_base)) {
2441 		if (SHMEM2_HAS(bp, dcc_support))
2442 			SHMEM2_WR(bp, dcc_support,
2443 				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2444 				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2445 		if (SHMEM2_HAS(bp, afex_driver_support))
2446 			SHMEM2_WR(bp, afex_driver_support,
2447 				  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2448 	}
2449 
2450 	/* Set AFEX default VLAN tag to an invalid value */
2451 	bp->afex_def_vlan_tag = -1;
2452 }
2453 
2454 /**
2455  * bnx2x_bz_fp - zero content of the fastpath structure.
2456  *
2457  * @bp:		driver handle
2458  * @index:	fastpath index to be zeroed
2459  *
2460  * Makes sure the contents of the bp->fp[index].napi is kept
2461  * intact.
2462  */
bnx2x_bz_fp(struct bnx2x * bp,int index)2463 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2464 {
2465 	struct bnx2x_fastpath *fp = &bp->fp[index];
2466 	int cos;
2467 	struct napi_struct orig_napi = fp->napi;
2468 	struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2469 
2470 	/* bzero bnx2x_fastpath contents */
2471 	if (fp->tpa_info)
2472 		memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2473 		       sizeof(struct bnx2x_agg_info));
2474 	memset(fp, 0, sizeof(*fp));
2475 
2476 	/* Restore the NAPI object as it has been already initialized */
2477 	fp->napi = orig_napi;
2478 	fp->tpa_info = orig_tpa_info;
2479 	fp->bp = bp;
2480 	fp->index = index;
2481 	if (IS_ETH_FP(fp))
2482 		fp->max_cos = bp->max_cos;
2483 	else
2484 		/* Special queues support only one CoS */
2485 		fp->max_cos = 1;
2486 
2487 	/* Init txdata pointers */
2488 	if (IS_FCOE_FP(fp))
2489 		fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2490 	if (IS_ETH_FP(fp))
2491 		for_each_cos_in_tx_queue(fp, cos)
2492 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2493 				BNX2X_NUM_ETH_QUEUES(bp) + index];
2494 
2495 	/* set the tpa flag for each queue. The tpa flag determines the queue
2496 	 * minimal size so it must be set prior to queue memory allocation
2497 	 */
2498 	if (bp->dev->features & NETIF_F_LRO)
2499 		fp->mode = TPA_MODE_LRO;
2500 	else if (bp->dev->features & NETIF_F_GRO_HW)
2501 		fp->mode = TPA_MODE_GRO;
2502 	else
2503 		fp->mode = TPA_MODE_DISABLED;
2504 
2505 	/* We don't want TPA if it's disabled in bp
2506 	 * or if this is an FCoE L2 ring.
2507 	 */
2508 	if (bp->disable_tpa || IS_FCOE_FP(fp))
2509 		fp->mode = TPA_MODE_DISABLED;
2510 }
2511 
bnx2x_set_os_driver_state(struct bnx2x * bp,u32 state)2512 void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state)
2513 {
2514 	u32 cur;
2515 
2516 	if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp))
2517 		return;
2518 
2519 	cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]);
2520 	DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n",
2521 	   cur, state);
2522 
2523 	SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state);
2524 }
2525 
bnx2x_load_cnic(struct bnx2x * bp)2526 int bnx2x_load_cnic(struct bnx2x *bp)
2527 {
2528 	int i, rc, port = BP_PORT(bp);
2529 
2530 	DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2531 
2532 	mutex_init(&bp->cnic_mutex);
2533 
2534 	if (IS_PF(bp)) {
2535 		rc = bnx2x_alloc_mem_cnic(bp);
2536 		if (rc) {
2537 			BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2538 			LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2539 		}
2540 	}
2541 
2542 	rc = bnx2x_alloc_fp_mem_cnic(bp);
2543 	if (rc) {
2544 		BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2545 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2546 	}
2547 
2548 	/* Update the number of queues with the cnic queues */
2549 	rc = bnx2x_set_real_num_queues(bp, 1);
2550 	if (rc) {
2551 		BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2552 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2553 	}
2554 
2555 	/* Add all CNIC NAPI objects */
2556 	bnx2x_add_all_napi_cnic(bp);
2557 	DP(NETIF_MSG_IFUP, "cnic napi added\n");
2558 	bnx2x_napi_enable_cnic(bp);
2559 
2560 	rc = bnx2x_init_hw_func_cnic(bp);
2561 	if (rc)
2562 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2563 
2564 	bnx2x_nic_init_cnic(bp);
2565 
2566 	if (IS_PF(bp)) {
2567 		/* Enable Timer scan */
2568 		REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2569 
2570 		/* setup cnic queues */
2571 		for_each_cnic_queue(bp, i) {
2572 			rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2573 			if (rc) {
2574 				BNX2X_ERR("Queue setup failed\n");
2575 				LOAD_ERROR_EXIT(bp, load_error_cnic2);
2576 			}
2577 		}
2578 	}
2579 
2580 	/* Initialize Rx filter. */
2581 	bnx2x_set_rx_mode_inner(bp);
2582 
2583 	/* re-read iscsi info */
2584 	bnx2x_get_iscsi_info(bp);
2585 	bnx2x_setup_cnic_irq_info(bp);
2586 	bnx2x_setup_cnic_info(bp);
2587 	bp->cnic_loaded = true;
2588 	if (bp->state == BNX2X_STATE_OPEN)
2589 		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2590 
2591 	DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2592 
2593 	return 0;
2594 
2595 #ifndef BNX2X_STOP_ON_ERROR
2596 load_error_cnic2:
2597 	/* Disable Timer scan */
2598 	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2599 
2600 load_error_cnic1:
2601 	bnx2x_napi_disable_cnic(bp);
2602 	/* Update the number of queues without the cnic queues */
2603 	if (bnx2x_set_real_num_queues(bp, 0))
2604 		BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2605 load_error_cnic0:
2606 	BNX2X_ERR("CNIC-related load failed\n");
2607 	bnx2x_free_fp_mem_cnic(bp);
2608 	bnx2x_free_mem_cnic(bp);
2609 	return rc;
2610 #endif /* ! BNX2X_STOP_ON_ERROR */
2611 }
2612 
2613 /* must be called with rtnl_lock */
bnx2x_nic_load(struct bnx2x * bp,int load_mode)2614 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2615 {
2616 	int port = BP_PORT(bp);
2617 	int i, rc = 0, load_code = 0;
2618 
2619 	DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2620 	DP(NETIF_MSG_IFUP,
2621 	   "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2622 
2623 #ifdef BNX2X_STOP_ON_ERROR
2624 	if (unlikely(bp->panic)) {
2625 		BNX2X_ERR("Can't load NIC when there is panic\n");
2626 		return -EPERM;
2627 	}
2628 #endif
2629 
2630 	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2631 
2632 	/* zero the structure w/o any lock, before SP handler is initialized */
2633 	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2634 	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2635 		&bp->last_reported_link.link_report_flags);
2636 
2637 	if (IS_PF(bp))
2638 		/* must be called before memory allocation and HW init */
2639 		bnx2x_ilt_set_info(bp);
2640 
2641 	/*
2642 	 * Zero fastpath structures preserving invariants like napi, which are
2643 	 * allocated only once, fp index, max_cos, bp pointer.
2644 	 * Also set fp->mode and txdata_ptr.
2645 	 */
2646 	DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2647 	for_each_queue(bp, i)
2648 		bnx2x_bz_fp(bp, i);
2649 	memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2650 				  bp->num_cnic_queues) *
2651 				  sizeof(struct bnx2x_fp_txdata));
2652 
2653 	bp->fcoe_init = false;
2654 
2655 	/* Set the receive queues buffer size */
2656 	bnx2x_set_rx_buf_size(bp);
2657 
2658 	if (IS_PF(bp)) {
2659 		rc = bnx2x_alloc_mem(bp);
2660 		if (rc) {
2661 			BNX2X_ERR("Unable to allocate bp memory\n");
2662 			return rc;
2663 		}
2664 	}
2665 
2666 	/* need to be done after alloc mem, since it's self adjusting to amount
2667 	 * of memory available for RSS queues
2668 	 */
2669 	rc = bnx2x_alloc_fp_mem(bp);
2670 	if (rc) {
2671 		BNX2X_ERR("Unable to allocate memory for fps\n");
2672 		LOAD_ERROR_EXIT(bp, load_error0);
2673 	}
2674 
2675 	/* Allocated memory for FW statistics  */
2676 	rc = bnx2x_alloc_fw_stats_mem(bp);
2677 	if (rc)
2678 		LOAD_ERROR_EXIT(bp, load_error0);
2679 
2680 	/* request pf to initialize status blocks */
2681 	if (IS_VF(bp)) {
2682 		rc = bnx2x_vfpf_init(bp);
2683 		if (rc)
2684 			LOAD_ERROR_EXIT(bp, load_error0);
2685 	}
2686 
2687 	/* As long as bnx2x_alloc_mem() may possibly update
2688 	 * bp->num_queues, bnx2x_set_real_num_queues() should always
2689 	 * come after it. At this stage cnic queues are not counted.
2690 	 */
2691 	rc = bnx2x_set_real_num_queues(bp, 0);
2692 	if (rc) {
2693 		BNX2X_ERR("Unable to set real_num_queues\n");
2694 		LOAD_ERROR_EXIT(bp, load_error0);
2695 	}
2696 
2697 	/* configure multi cos mappings in kernel.
2698 	 * this configuration may be overridden by a multi class queue
2699 	 * discipline or by a dcbx negotiation result.
2700 	 */
2701 	bnx2x_setup_tc(bp->dev, bp->max_cos);
2702 
2703 	/* Add all NAPI objects */
2704 	bnx2x_add_all_napi(bp);
2705 	DP(NETIF_MSG_IFUP, "napi added\n");
2706 	bnx2x_napi_enable(bp);
2707 
2708 	if (IS_PF(bp)) {
2709 		/* set pf load just before approaching the MCP */
2710 		bnx2x_set_pf_load(bp);
2711 
2712 		/* if mcp exists send load request and analyze response */
2713 		if (!BP_NOMCP(bp)) {
2714 			/* attempt to load pf */
2715 			rc = bnx2x_nic_load_request(bp, &load_code);
2716 			if (rc)
2717 				LOAD_ERROR_EXIT(bp, load_error1);
2718 
2719 			/* what did mcp say? */
2720 			rc = bnx2x_compare_fw_ver(bp, load_code, true);
2721 			if (rc) {
2722 				bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2723 				LOAD_ERROR_EXIT(bp, load_error2);
2724 			}
2725 		} else {
2726 			load_code = bnx2x_nic_load_no_mcp(bp, port);
2727 		}
2728 
2729 		/* mark pmf if applicable */
2730 		bnx2x_nic_load_pmf(bp, load_code);
2731 
2732 		/* Init Function state controlling object */
2733 		bnx2x__init_func_obj(bp);
2734 
2735 		/* Initialize HW */
2736 		rc = bnx2x_init_hw(bp, load_code);
2737 		if (rc) {
2738 			BNX2X_ERR("HW init failed, aborting\n");
2739 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2740 			LOAD_ERROR_EXIT(bp, load_error2);
2741 		}
2742 	}
2743 
2744 	bnx2x_pre_irq_nic_init(bp);
2745 
2746 	/* Connect to IRQs */
2747 	rc = bnx2x_setup_irqs(bp);
2748 	if (rc) {
2749 		BNX2X_ERR("setup irqs failed\n");
2750 		if (IS_PF(bp))
2751 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2752 		LOAD_ERROR_EXIT(bp, load_error2);
2753 	}
2754 
2755 	/* Init per-function objects */
2756 	if (IS_PF(bp)) {
2757 		/* Setup NIC internals and enable interrupts */
2758 		bnx2x_post_irq_nic_init(bp, load_code);
2759 
2760 		bnx2x_init_bp_objs(bp);
2761 		bnx2x_iov_nic_init(bp);
2762 
2763 		/* Set AFEX default VLAN tag to an invalid value */
2764 		bp->afex_def_vlan_tag = -1;
2765 		bnx2x_nic_load_afex_dcc(bp, load_code);
2766 		bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2767 		rc = bnx2x_func_start(bp);
2768 		if (rc) {
2769 			BNX2X_ERR("Function start failed!\n");
2770 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2771 
2772 			LOAD_ERROR_EXIT(bp, load_error3);
2773 		}
2774 
2775 		/* Send LOAD_DONE command to MCP */
2776 		if (!BP_NOMCP(bp)) {
2777 			load_code = bnx2x_fw_command(bp,
2778 						     DRV_MSG_CODE_LOAD_DONE, 0);
2779 			if (!load_code) {
2780 				BNX2X_ERR("MCP response failure, aborting\n");
2781 				rc = -EBUSY;
2782 				LOAD_ERROR_EXIT(bp, load_error3);
2783 			}
2784 		}
2785 
2786 		/* initialize FW coalescing state machines in RAM */
2787 		bnx2x_update_coalesce(bp);
2788 	}
2789 
2790 	/* setup the leading queue */
2791 	rc = bnx2x_setup_leading(bp);
2792 	if (rc) {
2793 		BNX2X_ERR("Setup leading failed!\n");
2794 		LOAD_ERROR_EXIT(bp, load_error3);
2795 	}
2796 
2797 	/* set up the rest of the queues */
2798 	for_each_nondefault_eth_queue(bp, i) {
2799 		if (IS_PF(bp))
2800 			rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2801 		else /* VF */
2802 			rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2803 		if (rc) {
2804 			BNX2X_ERR("Queue %d setup failed\n", i);
2805 			LOAD_ERROR_EXIT(bp, load_error3);
2806 		}
2807 	}
2808 
2809 	/* setup rss */
2810 	rc = bnx2x_init_rss(bp);
2811 	if (rc) {
2812 		BNX2X_ERR("PF RSS init failed\n");
2813 		LOAD_ERROR_EXIT(bp, load_error3);
2814 	}
2815 
2816 	/* Now when Clients are configured we are ready to work */
2817 	bp->state = BNX2X_STATE_OPEN;
2818 
2819 	/* Configure a ucast MAC */
2820 	if (IS_PF(bp))
2821 		rc = bnx2x_set_eth_mac(bp, true);
2822 	else /* vf */
2823 		rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2824 					   true);
2825 	if (rc) {
2826 		BNX2X_ERR("Setting Ethernet MAC failed\n");
2827 		LOAD_ERROR_EXIT(bp, load_error3);
2828 	}
2829 
2830 	if (IS_PF(bp) && bp->pending_max) {
2831 		bnx2x_update_max_mf_config(bp, bp->pending_max);
2832 		bp->pending_max = 0;
2833 	}
2834 
2835 	bp->force_link_down = false;
2836 	if (bp->port.pmf) {
2837 		rc = bnx2x_initial_phy_init(bp, load_mode);
2838 		if (rc)
2839 			LOAD_ERROR_EXIT(bp, load_error3);
2840 	}
2841 	bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2842 
2843 	/* Start fast path */
2844 
2845 	/* Re-configure vlan filters */
2846 	rc = bnx2x_vlan_reconfigure_vid(bp);
2847 	if (rc)
2848 		LOAD_ERROR_EXIT(bp, load_error3);
2849 
2850 	/* Initialize Rx filter. */
2851 	bnx2x_set_rx_mode_inner(bp);
2852 
2853 	if (bp->flags & PTP_SUPPORTED) {
2854 		bnx2x_register_phc(bp);
2855 		bnx2x_init_ptp(bp);
2856 		bnx2x_configure_ptp_filters(bp);
2857 	}
2858 	/* Start Tx */
2859 	switch (load_mode) {
2860 	case LOAD_NORMAL:
2861 		/* Tx queue should be only re-enabled */
2862 		netif_tx_wake_all_queues(bp->dev);
2863 		break;
2864 
2865 	case LOAD_OPEN:
2866 		netif_tx_start_all_queues(bp->dev);
2867 		smp_mb__after_atomic();
2868 		break;
2869 
2870 	case LOAD_DIAG:
2871 	case LOAD_LOOPBACK_EXT:
2872 		bp->state = BNX2X_STATE_DIAG;
2873 		break;
2874 
2875 	default:
2876 		break;
2877 	}
2878 
2879 	if (bp->port.pmf)
2880 		bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2881 	else
2882 		bnx2x__link_status_update(bp);
2883 
2884 	/* start the timer */
2885 	mod_timer(&bp->timer, jiffies + bp->current_interval);
2886 
2887 	if (CNIC_ENABLED(bp))
2888 		bnx2x_load_cnic(bp);
2889 
2890 	if (IS_PF(bp))
2891 		bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2892 
2893 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2894 		/* mark driver is loaded in shmem2 */
2895 		u32 val;
2896 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2897 		val &= ~DRV_FLAGS_MTU_MASK;
2898 		val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT);
2899 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2900 			  val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2901 			  DRV_FLAGS_CAPABILITIES_LOADED_L2);
2902 	}
2903 
2904 	/* Wait for all pending SP commands to complete */
2905 	if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2906 		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2907 		bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2908 		return -EBUSY;
2909 	}
2910 
2911 	/* Update driver data for On-Chip MFW dump. */
2912 	if (IS_PF(bp))
2913 		bnx2x_update_mfw_dump(bp);
2914 
2915 	/* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2916 	if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2917 		bnx2x_dcbx_init(bp, false);
2918 
2919 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2920 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE);
2921 
2922 	DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2923 
2924 	return 0;
2925 
2926 #ifndef BNX2X_STOP_ON_ERROR
2927 load_error3:
2928 	if (IS_PF(bp)) {
2929 		bnx2x_int_disable_sync(bp, 1);
2930 
2931 		/* Clean queueable objects */
2932 		bnx2x_squeeze_objects(bp);
2933 	}
2934 
2935 	/* Free SKBs, SGEs, TPA pool and driver internals */
2936 	bnx2x_free_skbs(bp);
2937 	for_each_rx_queue(bp, i)
2938 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2939 
2940 	/* Release IRQs */
2941 	bnx2x_free_irq(bp);
2942 load_error2:
2943 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
2944 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2945 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2946 	}
2947 
2948 	bp->port.pmf = 0;
2949 load_error1:
2950 	bnx2x_napi_disable(bp);
2951 	bnx2x_del_all_napi(bp);
2952 
2953 	/* clear pf_load status, as it was already set */
2954 	if (IS_PF(bp))
2955 		bnx2x_clear_pf_load(bp);
2956 load_error0:
2957 	bnx2x_free_fw_stats_mem(bp);
2958 	bnx2x_free_fp_mem(bp);
2959 	bnx2x_free_mem(bp);
2960 
2961 	return rc;
2962 #endif /* ! BNX2X_STOP_ON_ERROR */
2963 }
2964 
bnx2x_drain_tx_queues(struct bnx2x * bp)2965 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2966 {
2967 	u8 rc = 0, cos, i;
2968 
2969 	/* Wait until tx fastpath tasks complete */
2970 	for_each_tx_queue(bp, i) {
2971 		struct bnx2x_fastpath *fp = &bp->fp[i];
2972 
2973 		for_each_cos_in_tx_queue(fp, cos)
2974 			rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2975 		if (rc)
2976 			return rc;
2977 	}
2978 	return 0;
2979 }
2980 
2981 /* must be called with rtnl_lock */
bnx2x_nic_unload(struct bnx2x * bp,int unload_mode,bool keep_link)2982 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2983 {
2984 	int i;
2985 	bool global = false;
2986 
2987 	DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2988 
2989 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2990 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED);
2991 
2992 	/* mark driver is unloaded in shmem2 */
2993 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2994 		u32 val;
2995 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2996 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2997 			  val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2998 	}
2999 
3000 	if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
3001 	    (bp->state == BNX2X_STATE_CLOSED ||
3002 	     bp->state == BNX2X_STATE_ERROR)) {
3003 		/* We can get here if the driver has been unloaded
3004 		 * during parity error recovery and is either waiting for a
3005 		 * leader to complete or for other functions to unload and
3006 		 * then ifdown has been issued. In this case we want to
3007 		 * unload and let other functions to complete a recovery
3008 		 * process.
3009 		 */
3010 		bp->recovery_state = BNX2X_RECOVERY_DONE;
3011 		bp->is_leader = 0;
3012 		bnx2x_release_leader_lock(bp);
3013 		smp_mb();
3014 
3015 		DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
3016 		BNX2X_ERR("Can't unload in closed or error state\n");
3017 		return -EINVAL;
3018 	}
3019 
3020 	/* Nothing to do during unload if previous bnx2x_nic_load()
3021 	 * have not completed successfully - all resources are released.
3022 	 *
3023 	 * we can get here only after unsuccessful ndo_* callback, during which
3024 	 * dev->IFF_UP flag is still on.
3025 	 */
3026 	if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3027 		return 0;
3028 
3029 	/* It's important to set the bp->state to the value different from
3030 	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3031 	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3032 	 */
3033 	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3034 	smp_mb();
3035 
3036 	/* indicate to VFs that the PF is going down */
3037 	bnx2x_iov_channel_down(bp);
3038 
3039 	if (CNIC_LOADED(bp))
3040 		bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3041 
3042 	/* Stop Tx */
3043 	bnx2x_tx_disable(bp);
3044 	netdev_reset_tc(bp->dev);
3045 
3046 	bp->rx_mode = BNX2X_RX_MODE_NONE;
3047 
3048 	del_timer_sync(&bp->timer);
3049 
3050 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
3051 		/* Set ALWAYS_ALIVE bit in shmem */
3052 		bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3053 		bnx2x_drv_pulse(bp);
3054 		bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3055 		bnx2x_save_statistics(bp);
3056 	}
3057 
3058 	/* wait till consumers catch up with producers in all queues.
3059 	 * If we're recovering, FW can't write to host so no reason
3060 	 * to wait for the queues to complete all Tx.
3061 	 */
3062 	if (unload_mode != UNLOAD_RECOVERY)
3063 		bnx2x_drain_tx_queues(bp);
3064 
3065 	/* if VF indicate to PF this function is going down (PF will delete sp
3066 	 * elements and clear initializations
3067 	 */
3068 	if (IS_VF(bp)) {
3069 		bnx2x_clear_vlan_info(bp);
3070 		bnx2x_vfpf_close_vf(bp);
3071 	} else if (unload_mode != UNLOAD_RECOVERY) {
3072 		/* if this is a normal/close unload need to clean up chip*/
3073 		bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3074 	} else {
3075 		/* Send the UNLOAD_REQUEST to the MCP */
3076 		bnx2x_send_unload_req(bp, unload_mode);
3077 
3078 		/* Prevent transactions to host from the functions on the
3079 		 * engine that doesn't reset global blocks in case of global
3080 		 * attention once global blocks are reset and gates are opened
3081 		 * (the engine which leader will perform the recovery
3082 		 * last).
3083 		 */
3084 		if (!CHIP_IS_E1x(bp))
3085 			bnx2x_pf_disable(bp);
3086 
3087 		/* Disable HW interrupts, NAPI */
3088 		bnx2x_netif_stop(bp, 1);
3089 		/* Delete all NAPI objects */
3090 		bnx2x_del_all_napi(bp);
3091 		if (CNIC_LOADED(bp))
3092 			bnx2x_del_all_napi_cnic(bp);
3093 		/* Release IRQs */
3094 		bnx2x_free_irq(bp);
3095 
3096 		/* Report UNLOAD_DONE to MCP */
3097 		bnx2x_send_unload_done(bp, false);
3098 	}
3099 
3100 	/*
3101 	 * At this stage no more interrupts will arrive so we may safely clean
3102 	 * the queueable objects here in case they failed to get cleaned so far.
3103 	 */
3104 	if (IS_PF(bp))
3105 		bnx2x_squeeze_objects(bp);
3106 
3107 	/* There should be no more pending SP commands at this stage */
3108 	bp->sp_state = 0;
3109 
3110 	bp->port.pmf = 0;
3111 
3112 	/* clear pending work in rtnl task */
3113 	bp->sp_rtnl_state = 0;
3114 	smp_mb();
3115 
3116 	/* Free SKBs, SGEs, TPA pool and driver internals */
3117 	bnx2x_free_skbs(bp);
3118 	if (CNIC_LOADED(bp))
3119 		bnx2x_free_skbs_cnic(bp);
3120 	for_each_rx_queue(bp, i)
3121 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3122 
3123 	bnx2x_free_fp_mem(bp);
3124 	if (CNIC_LOADED(bp))
3125 		bnx2x_free_fp_mem_cnic(bp);
3126 
3127 	if (IS_PF(bp)) {
3128 		if (CNIC_LOADED(bp))
3129 			bnx2x_free_mem_cnic(bp);
3130 	}
3131 	bnx2x_free_mem(bp);
3132 
3133 	bp->state = BNX2X_STATE_CLOSED;
3134 	bp->cnic_loaded = false;
3135 
3136 	/* Clear driver version indication in shmem */
3137 	if (IS_PF(bp) && !BP_NOMCP(bp))
3138 		bnx2x_update_mng_version(bp);
3139 
3140 	/* Check if there are pending parity attentions. If there are - set
3141 	 * RECOVERY_IN_PROGRESS.
3142 	 */
3143 	if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3144 		bnx2x_set_reset_in_progress(bp);
3145 
3146 		/* Set RESET_IS_GLOBAL if needed */
3147 		if (global)
3148 			bnx2x_set_reset_global(bp);
3149 	}
3150 
3151 	/* The last driver must disable a "close the gate" if there is no
3152 	 * parity attention or "process kill" pending.
3153 	 */
3154 	if (IS_PF(bp) &&
3155 	    !bnx2x_clear_pf_load(bp) &&
3156 	    bnx2x_reset_is_done(bp, BP_PATH(bp)))
3157 		bnx2x_disable_close_the_gate(bp);
3158 
3159 	DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3160 
3161 	return 0;
3162 }
3163 
bnx2x_set_power_state(struct bnx2x * bp,pci_power_t state)3164 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3165 {
3166 	u16 pmcsr;
3167 
3168 	/* If there is no power capability, silently succeed */
3169 	if (!bp->pdev->pm_cap) {
3170 		BNX2X_DEV_INFO("No power capability. Breaking.\n");
3171 		return 0;
3172 	}
3173 
3174 	pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3175 
3176 	switch (state) {
3177 	case PCI_D0:
3178 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3179 				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3180 				       PCI_PM_CTRL_PME_STATUS));
3181 
3182 		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3183 			/* delay required during transition out of D3hot */
3184 			msleep(20);
3185 		break;
3186 
3187 	case PCI_D3hot:
3188 		/* If there are other clients above don't
3189 		   shut down the power */
3190 		if (atomic_read(&bp->pdev->enable_cnt) != 1)
3191 			return 0;
3192 		/* Don't shut down the power for emulation and FPGA */
3193 		if (CHIP_REV_IS_SLOW(bp))
3194 			return 0;
3195 
3196 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3197 		pmcsr |= 3;
3198 
3199 		if (bp->wol)
3200 			pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3201 
3202 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3203 				      pmcsr);
3204 
3205 		/* No more memory access after this point until
3206 		* device is brought back to D0.
3207 		*/
3208 		break;
3209 
3210 	default:
3211 		dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3212 		return -EINVAL;
3213 	}
3214 	return 0;
3215 }
3216 
3217 /*
3218  * net_device service functions
3219  */
bnx2x_poll(struct napi_struct * napi,int budget)3220 static int bnx2x_poll(struct napi_struct *napi, int budget)
3221 {
3222 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3223 						 napi);
3224 	struct bnx2x *bp = fp->bp;
3225 	int rx_work_done;
3226 	u8 cos;
3227 
3228 #ifdef BNX2X_STOP_ON_ERROR
3229 	if (unlikely(bp->panic)) {
3230 		napi_complete(napi);
3231 		return 0;
3232 	}
3233 #endif
3234 	for_each_cos_in_tx_queue(fp, cos)
3235 		if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3236 			bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3237 
3238 	rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0;
3239 
3240 	if (rx_work_done < budget) {
3241 		/* No need to update SB for FCoE L2 ring as long as
3242 		 * it's connected to the default SB and the SB
3243 		 * has been updated when NAPI was scheduled.
3244 		 */
3245 		if (IS_FCOE_FP(fp)) {
3246 			napi_complete_done(napi, rx_work_done);
3247 		} else {
3248 			bnx2x_update_fpsb_idx(fp);
3249 			/* bnx2x_has_rx_work() reads the status block,
3250 			 * thus we need to ensure that status block indices
3251 			 * have been actually read (bnx2x_update_fpsb_idx)
3252 			 * prior to this check (bnx2x_has_rx_work) so that
3253 			 * we won't write the "newer" value of the status block
3254 			 * to IGU (if there was a DMA right after
3255 			 * bnx2x_has_rx_work and if there is no rmb, the memory
3256 			 * reading (bnx2x_update_fpsb_idx) may be postponed
3257 			 * to right before bnx2x_ack_sb). In this case there
3258 			 * will never be another interrupt until there is
3259 			 * another update of the status block, while there
3260 			 * is still unhandled work.
3261 			 */
3262 			rmb();
3263 
3264 			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3265 				if (napi_complete_done(napi, rx_work_done)) {
3266 					/* Re-enable interrupts */
3267 					DP(NETIF_MSG_RX_STATUS,
3268 					   "Update index to %d\n", fp->fp_hc_idx);
3269 					bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3270 						     le16_to_cpu(fp->fp_hc_idx),
3271 						     IGU_INT_ENABLE, 1);
3272 				}
3273 			} else {
3274 				rx_work_done = budget;
3275 			}
3276 		}
3277 	}
3278 
3279 	return rx_work_done;
3280 }
3281 
3282 /* we split the first BD into headers and data BDs
3283  * to ease the pain of our fellow microcode engineers
3284  * we use one mapping for both BDs
3285  */
bnx2x_tx_split(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata,struct sw_tx_bd * tx_buf,struct eth_tx_start_bd ** tx_bd,u16 hlen,u16 bd_prod)3286 static u16 bnx2x_tx_split(struct bnx2x *bp,
3287 			  struct bnx2x_fp_txdata *txdata,
3288 			  struct sw_tx_bd *tx_buf,
3289 			  struct eth_tx_start_bd **tx_bd, u16 hlen,
3290 			  u16 bd_prod)
3291 {
3292 	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3293 	struct eth_tx_bd *d_tx_bd;
3294 	dma_addr_t mapping;
3295 	int old_len = le16_to_cpu(h_tx_bd->nbytes);
3296 
3297 	/* first fix first BD */
3298 	h_tx_bd->nbytes = cpu_to_le16(hlen);
3299 
3300 	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d (%x:%x)\n",
3301 	   h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3302 
3303 	/* now get a new data BD
3304 	 * (after the pbd) and fill it */
3305 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3306 	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3307 
3308 	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3309 			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3310 
3311 	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3312 	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3313 	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3314 
3315 	/* this marks the BD as one that has no individual mapping */
3316 	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3317 
3318 	DP(NETIF_MSG_TX_QUEUED,
3319 	   "TSO split data size is %d (%x:%x)\n",
3320 	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3321 
3322 	/* update tx_bd */
3323 	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3324 
3325 	return bd_prod;
3326 }
3327 
3328 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3329 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
bnx2x_csum_fix(unsigned char * t_header,u16 csum,s8 fix)3330 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3331 {
3332 	__sum16 tsum = (__force __sum16) csum;
3333 
3334 	if (fix > 0)
3335 		tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3336 				  csum_partial(t_header - fix, fix, 0)));
3337 
3338 	else if (fix < 0)
3339 		tsum = ~csum_fold(csum_add((__force __wsum) csum,
3340 				  csum_partial(t_header, -fix, 0)));
3341 
3342 	return bswab16(tsum);
3343 }
3344 
bnx2x_xmit_type(struct bnx2x * bp,struct sk_buff * skb)3345 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3346 {
3347 	u32 rc;
3348 	__u8 prot = 0;
3349 	__be16 protocol;
3350 
3351 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3352 		return XMIT_PLAIN;
3353 
3354 	protocol = vlan_get_protocol(skb);
3355 	if (protocol == htons(ETH_P_IPV6)) {
3356 		rc = XMIT_CSUM_V6;
3357 		prot = ipv6_hdr(skb)->nexthdr;
3358 	} else {
3359 		rc = XMIT_CSUM_V4;
3360 		prot = ip_hdr(skb)->protocol;
3361 	}
3362 
3363 	if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3364 		if (inner_ip_hdr(skb)->version == 6) {
3365 			rc |= XMIT_CSUM_ENC_V6;
3366 			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3367 				rc |= XMIT_CSUM_TCP;
3368 		} else {
3369 			rc |= XMIT_CSUM_ENC_V4;
3370 			if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3371 				rc |= XMIT_CSUM_TCP;
3372 		}
3373 	}
3374 	if (prot == IPPROTO_TCP)
3375 		rc |= XMIT_CSUM_TCP;
3376 
3377 	if (skb_is_gso(skb)) {
3378 		if (skb_is_gso_v6(skb)) {
3379 			rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3380 			if (rc & XMIT_CSUM_ENC)
3381 				rc |= XMIT_GSO_ENC_V6;
3382 		} else {
3383 			rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3384 			if (rc & XMIT_CSUM_ENC)
3385 				rc |= XMIT_GSO_ENC_V4;
3386 		}
3387 	}
3388 
3389 	return rc;
3390 }
3391 
3392 /* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */
3393 #define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS         4
3394 
3395 /* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3396 #define BNX2X_NUM_TSO_WIN_SUB_BDS               3
3397 
3398 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3399 /* check if packet requires linearization (packet is too fragmented)
3400    no need to check fragmentation if page size > 8K (there will be no
3401    violation to FW restrictions) */
bnx2x_pkt_req_lin(struct bnx2x * bp,struct sk_buff * skb,u32 xmit_type)3402 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3403 			     u32 xmit_type)
3404 {
3405 	int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS;
3406 	int to_copy = 0, hlen = 0;
3407 
3408 	if (xmit_type & XMIT_GSO_ENC)
3409 		num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS;
3410 
3411 	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) {
3412 		if (xmit_type & XMIT_GSO) {
3413 			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3414 			int wnd_size = MAX_FETCH_BD - num_tso_win_sub;
3415 			/* Number of windows to check */
3416 			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3417 			int wnd_idx = 0;
3418 			int frag_idx = 0;
3419 			u32 wnd_sum = 0;
3420 
3421 			/* Headers length */
3422 			if (xmit_type & XMIT_GSO_ENC)
3423 				hlen = skb_inner_tcp_all_headers(skb);
3424 			else
3425 				hlen = skb_tcp_all_headers(skb);
3426 
3427 			/* Amount of data (w/o headers) on linear part of SKB*/
3428 			first_bd_sz = skb_headlen(skb) - hlen;
3429 
3430 			wnd_sum  = first_bd_sz;
3431 
3432 			/* Calculate the first sum - it's special */
3433 			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3434 				wnd_sum +=
3435 					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3436 
3437 			/* If there was data on linear skb data - check it */
3438 			if (first_bd_sz > 0) {
3439 				if (unlikely(wnd_sum < lso_mss)) {
3440 					to_copy = 1;
3441 					goto exit_lbl;
3442 				}
3443 
3444 				wnd_sum -= first_bd_sz;
3445 			}
3446 
3447 			/* Others are easier: run through the frag list and
3448 			   check all windows */
3449 			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3450 				wnd_sum +=
3451 			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3452 
3453 				if (unlikely(wnd_sum < lso_mss)) {
3454 					to_copy = 1;
3455 					break;
3456 				}
3457 				wnd_sum -=
3458 					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3459 			}
3460 		} else {
3461 			/* in non-LSO too fragmented packet should always
3462 			   be linearized */
3463 			to_copy = 1;
3464 		}
3465 	}
3466 
3467 exit_lbl:
3468 	if (unlikely(to_copy))
3469 		DP(NETIF_MSG_TX_QUEUED,
3470 		   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3471 		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3472 		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3473 
3474 	return to_copy;
3475 }
3476 #endif
3477 
3478 /**
3479  * bnx2x_set_pbd_gso - update PBD in GSO case.
3480  *
3481  * @skb:	packet skb
3482  * @pbd:	parse BD
3483  * @xmit_type:	xmit flags
3484  */
bnx2x_set_pbd_gso(struct sk_buff * skb,struct eth_tx_parse_bd_e1x * pbd,u32 xmit_type)3485 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3486 			      struct eth_tx_parse_bd_e1x *pbd,
3487 			      u32 xmit_type)
3488 {
3489 	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3490 	pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3491 	pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3492 
3493 	if (xmit_type & XMIT_GSO_V4) {
3494 		pbd->ip_id = bswab16(ip_hdr(skb)->id);
3495 		pbd->tcp_pseudo_csum =
3496 			bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3497 						   ip_hdr(skb)->daddr,
3498 						   0, IPPROTO_TCP, 0));
3499 	} else {
3500 		pbd->tcp_pseudo_csum =
3501 			bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3502 						 &ipv6_hdr(skb)->daddr,
3503 						 0, IPPROTO_TCP, 0));
3504 	}
3505 
3506 	pbd->global_data |=
3507 		cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3508 }
3509 
3510 /**
3511  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3512  *
3513  * @bp:			driver handle
3514  * @skb:		packet skb
3515  * @parsing_data:	data to be updated
3516  * @xmit_type:		xmit flags
3517  *
3518  * 57712/578xx related, when skb has encapsulation
3519  */
bnx2x_set_pbd_csum_enc(struct bnx2x * bp,struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3520 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3521 				 u32 *parsing_data, u32 xmit_type)
3522 {
3523 	*parsing_data |=
3524 		((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3525 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3526 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3527 
3528 	if (xmit_type & XMIT_CSUM_TCP) {
3529 		*parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3530 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3531 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3532 
3533 		return skb_inner_tcp_all_headers(skb);
3534 	}
3535 
3536 	/* We support checksum offload for TCP and UDP only.
3537 	 * No need to pass the UDP header length - it's a constant.
3538 	 */
3539 	return skb_inner_transport_offset(skb) + sizeof(struct udphdr);
3540 }
3541 
3542 /**
3543  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3544  *
3545  * @bp:			driver handle
3546  * @skb:		packet skb
3547  * @parsing_data:	data to be updated
3548  * @xmit_type:		xmit flags
3549  *
3550  * 57712/578xx related
3551  */
bnx2x_set_pbd_csum_e2(struct bnx2x * bp,struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3552 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3553 				u32 *parsing_data, u32 xmit_type)
3554 {
3555 	*parsing_data |=
3556 		((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3557 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3558 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3559 
3560 	if (xmit_type & XMIT_CSUM_TCP) {
3561 		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3562 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3563 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3564 
3565 		return skb_tcp_all_headers(skb);
3566 	}
3567 	/* We support checksum offload for TCP and UDP only.
3568 	 * No need to pass the UDP header length - it's a constant.
3569 	 */
3570 	return skb_transport_offset(skb) + sizeof(struct udphdr);
3571 }
3572 
3573 /* set FW indication according to inner or outer protocols if tunneled */
bnx2x_set_sbd_csum(struct bnx2x * bp,struct sk_buff * skb,struct eth_tx_start_bd * tx_start_bd,u32 xmit_type)3574 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3575 			       struct eth_tx_start_bd *tx_start_bd,
3576 			       u32 xmit_type)
3577 {
3578 	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3579 
3580 	if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3581 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3582 
3583 	if (!(xmit_type & XMIT_CSUM_TCP))
3584 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3585 }
3586 
3587 /**
3588  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3589  *
3590  * @bp:		driver handle
3591  * @skb:	packet skb
3592  * @pbd:	parse BD to be updated
3593  * @xmit_type:	xmit flags
3594  */
bnx2x_set_pbd_csum(struct bnx2x * bp,struct sk_buff * skb,struct eth_tx_parse_bd_e1x * pbd,u32 xmit_type)3595 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3596 			     struct eth_tx_parse_bd_e1x *pbd,
3597 			     u32 xmit_type)
3598 {
3599 	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3600 
3601 	/* for now NS flag is not used in Linux */
3602 	pbd->global_data =
3603 		cpu_to_le16(hlen |
3604 			    ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3605 			     ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3606 
3607 	pbd->ip_hlen_w = (skb_transport_header(skb) -
3608 			skb_network_header(skb)) >> 1;
3609 
3610 	hlen += pbd->ip_hlen_w;
3611 
3612 	/* We support checksum offload for TCP and UDP only */
3613 	if (xmit_type & XMIT_CSUM_TCP)
3614 		hlen += tcp_hdrlen(skb) / 2;
3615 	else
3616 		hlen += sizeof(struct udphdr) / 2;
3617 
3618 	pbd->total_hlen_w = cpu_to_le16(hlen);
3619 	hlen = hlen*2;
3620 
3621 	if (xmit_type & XMIT_CSUM_TCP) {
3622 		pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3623 
3624 	} else {
3625 		s8 fix = SKB_CS_OFF(skb); /* signed! */
3626 
3627 		DP(NETIF_MSG_TX_QUEUED,
3628 		   "hlen %d  fix %d  csum before fix %x\n",
3629 		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3630 
3631 		/* HW bug: fixup the CSUM */
3632 		pbd->tcp_pseudo_csum =
3633 			bnx2x_csum_fix(skb_transport_header(skb),
3634 				       SKB_CS(skb), fix);
3635 
3636 		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3637 		   pbd->tcp_pseudo_csum);
3638 	}
3639 
3640 	return hlen;
3641 }
3642 
bnx2x_update_pbds_gso_enc(struct sk_buff * skb,struct eth_tx_parse_bd_e2 * pbd_e2,struct eth_tx_parse_2nd_bd * pbd2,u16 * global_data,u32 xmit_type)3643 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3644 				      struct eth_tx_parse_bd_e2 *pbd_e2,
3645 				      struct eth_tx_parse_2nd_bd *pbd2,
3646 				      u16 *global_data,
3647 				      u32 xmit_type)
3648 {
3649 	u16 hlen_w = 0;
3650 	u8 outerip_off, outerip_len = 0;
3651 
3652 	/* from outer IP to transport */
3653 	hlen_w = (skb_inner_transport_header(skb) -
3654 		  skb_network_header(skb)) >> 1;
3655 
3656 	/* transport len */
3657 	hlen_w += inner_tcp_hdrlen(skb) >> 1;
3658 
3659 	pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3660 
3661 	/* outer IP header info */
3662 	if (xmit_type & XMIT_CSUM_V4) {
3663 		struct iphdr *iph = ip_hdr(skb);
3664 		u32 csum = (__force u32)(~iph->check) -
3665 			   (__force u32)iph->tot_len -
3666 			   (__force u32)iph->frag_off;
3667 
3668 		outerip_len = iph->ihl << 1;
3669 
3670 		pbd2->fw_ip_csum_wo_len_flags_frag =
3671 			bswab16(csum_fold((__force __wsum)csum));
3672 	} else {
3673 		pbd2->fw_ip_hdr_to_payload_w =
3674 			hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3675 		pbd_e2->data.tunnel_data.flags |=
3676 			ETH_TUNNEL_DATA_IPV6_OUTER;
3677 	}
3678 
3679 	pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3680 
3681 	pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3682 
3683 	/* inner IP header info */
3684 	if (xmit_type & XMIT_CSUM_ENC_V4) {
3685 		pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3686 
3687 		pbd_e2->data.tunnel_data.pseudo_csum =
3688 			bswab16(~csum_tcpudp_magic(
3689 					inner_ip_hdr(skb)->saddr,
3690 					inner_ip_hdr(skb)->daddr,
3691 					0, IPPROTO_TCP, 0));
3692 	} else {
3693 		pbd_e2->data.tunnel_data.pseudo_csum =
3694 			bswab16(~csum_ipv6_magic(
3695 					&inner_ipv6_hdr(skb)->saddr,
3696 					&inner_ipv6_hdr(skb)->daddr,
3697 					0, IPPROTO_TCP, 0));
3698 	}
3699 
3700 	outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3701 
3702 	*global_data |=
3703 		outerip_off |
3704 		(outerip_len <<
3705 			ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3706 		((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3707 			ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3708 
3709 	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3710 		SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3711 		pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3712 	}
3713 }
3714 
bnx2x_set_ipv6_ext_e2(struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3715 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3716 					 u32 xmit_type)
3717 {
3718 	struct ipv6hdr *ipv6;
3719 
3720 	if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3721 		return;
3722 
3723 	if (xmit_type & XMIT_GSO_ENC_V6)
3724 		ipv6 = inner_ipv6_hdr(skb);
3725 	else /* XMIT_GSO_V6 */
3726 		ipv6 = ipv6_hdr(skb);
3727 
3728 	if (ipv6->nexthdr == NEXTHDR_IPV6)
3729 		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3730 }
3731 
3732 /* called with netif_tx_lock
3733  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3734  * netif_wake_queue()
3735  */
bnx2x_start_xmit(struct sk_buff * skb,struct net_device * dev)3736 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3737 {
3738 	struct bnx2x *bp = netdev_priv(dev);
3739 
3740 	struct netdev_queue *txq;
3741 	struct bnx2x_fp_txdata *txdata;
3742 	struct sw_tx_bd *tx_buf;
3743 	struct eth_tx_start_bd *tx_start_bd, *first_bd;
3744 	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3745 	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3746 	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3747 	struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3748 	u32 pbd_e2_parsing_data = 0;
3749 	u16 pkt_prod, bd_prod;
3750 	int nbd, txq_index;
3751 	dma_addr_t mapping;
3752 	u32 xmit_type = bnx2x_xmit_type(bp, skb);
3753 	int i;
3754 	u8 hlen = 0;
3755 	__le16 pkt_size = 0;
3756 	struct ethhdr *eth;
3757 	u8 mac_type = UNICAST_ADDRESS;
3758 
3759 #ifdef BNX2X_STOP_ON_ERROR
3760 	if (unlikely(bp->panic))
3761 		return NETDEV_TX_BUSY;
3762 #endif
3763 
3764 	txq_index = skb_get_queue_mapping(skb);
3765 	txq = netdev_get_tx_queue(dev, txq_index);
3766 
3767 	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3768 
3769 	txdata = &bp->bnx2x_txq[txq_index];
3770 
3771 	/* enable this debug print to view the transmission queue being used
3772 	DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3773 	   txq_index, fp_index, txdata_index); */
3774 
3775 	/* enable this debug print to view the transmission details
3776 	DP(NETIF_MSG_TX_QUEUED,
3777 	   "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3778 	   txdata->cid, fp_index, txdata_index, txdata, fp); */
3779 
3780 	if (unlikely(bnx2x_tx_avail(bp, txdata) <
3781 			skb_shinfo(skb)->nr_frags +
3782 			BDS_PER_TX_PKT +
3783 			NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3784 		/* Handle special storage cases separately */
3785 		if (txdata->tx_ring_size == 0) {
3786 			struct bnx2x_eth_q_stats *q_stats =
3787 				bnx2x_fp_qstats(bp, txdata->parent_fp);
3788 			q_stats->driver_filtered_tx_pkt++;
3789 			dev_kfree_skb(skb);
3790 			return NETDEV_TX_OK;
3791 		}
3792 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3793 		netif_tx_stop_queue(txq);
3794 		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3795 
3796 		return NETDEV_TX_BUSY;
3797 	}
3798 
3799 	DP(NETIF_MSG_TX_QUEUED,
3800 	   "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3801 	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3802 	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3803 	   skb->len);
3804 
3805 	eth = (struct ethhdr *)skb->data;
3806 
3807 	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
3808 	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3809 		if (is_broadcast_ether_addr(eth->h_dest))
3810 			mac_type = BROADCAST_ADDRESS;
3811 		else
3812 			mac_type = MULTICAST_ADDRESS;
3813 	}
3814 
3815 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3816 	/* First, check if we need to linearize the skb (due to FW
3817 	   restrictions). No need to check fragmentation if page size > 8K
3818 	   (there will be no violation to FW restrictions) */
3819 	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3820 		/* Statistics of linearization */
3821 		bp->lin_cnt++;
3822 		if (skb_linearize(skb) != 0) {
3823 			DP(NETIF_MSG_TX_QUEUED,
3824 			   "SKB linearization failed - silently dropping this SKB\n");
3825 			dev_kfree_skb_any(skb);
3826 			return NETDEV_TX_OK;
3827 		}
3828 	}
3829 #endif
3830 	/* Map skb linear data for DMA */
3831 	mapping = dma_map_single(&bp->pdev->dev, skb->data,
3832 				 skb_headlen(skb), DMA_TO_DEVICE);
3833 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3834 		DP(NETIF_MSG_TX_QUEUED,
3835 		   "SKB mapping failed - silently dropping this SKB\n");
3836 		dev_kfree_skb_any(skb);
3837 		return NETDEV_TX_OK;
3838 	}
3839 	/*
3840 	Please read carefully. First we use one BD which we mark as start,
3841 	then we have a parsing info BD (used for TSO or xsum),
3842 	and only then we have the rest of the TSO BDs.
3843 	(don't forget to mark the last one as last,
3844 	and to unmap only AFTER you write to the BD ...)
3845 	And above all, all pdb sizes are in words - NOT DWORDS!
3846 	*/
3847 
3848 	/* get current pkt produced now - advance it just before sending packet
3849 	 * since mapping of pages may fail and cause packet to be dropped
3850 	 */
3851 	pkt_prod = txdata->tx_pkt_prod;
3852 	bd_prod = TX_BD(txdata->tx_bd_prod);
3853 
3854 	/* get a tx_buf and first BD
3855 	 * tx_start_bd may be changed during SPLIT,
3856 	 * but first_bd will always stay first
3857 	 */
3858 	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3859 	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3860 	first_bd = tx_start_bd;
3861 
3862 	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3863 
3864 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3865 		if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3866 			bp->eth_stats.ptp_skip_tx_ts++;
3867 			BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3868 		} else if (bp->ptp_tx_skb) {
3869 			bp->eth_stats.ptp_skip_tx_ts++;
3870 			netdev_err_once(bp->dev,
3871 					"Device supports only a single outstanding packet to timestamp, this packet won't be timestamped\n");
3872 		} else {
3873 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3874 			/* schedule check for Tx timestamp */
3875 			bp->ptp_tx_skb = skb_get(skb);
3876 			bp->ptp_tx_start = jiffies;
3877 			schedule_work(&bp->ptp_task);
3878 		}
3879 	}
3880 
3881 	/* header nbd: indirectly zero other flags! */
3882 	tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3883 
3884 	/* remember the first BD of the packet */
3885 	tx_buf->first_bd = txdata->tx_bd_prod;
3886 	tx_buf->skb = skb;
3887 	tx_buf->flags = 0;
3888 
3889 	DP(NETIF_MSG_TX_QUEUED,
3890 	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3891 	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3892 
3893 	if (skb_vlan_tag_present(skb)) {
3894 		tx_start_bd->vlan_or_ethertype =
3895 		    cpu_to_le16(skb_vlan_tag_get(skb));
3896 		tx_start_bd->bd_flags.as_bitfield |=
3897 		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3898 	} else {
3899 		/* when transmitting in a vf, start bd must hold the ethertype
3900 		 * for fw to enforce it
3901 		 */
3902 		u16 vlan_tci = 0;
3903 #ifndef BNX2X_STOP_ON_ERROR
3904 		if (IS_VF(bp)) {
3905 #endif
3906 			/* Still need to consider inband vlan for enforced */
3907 			if (__vlan_get_tag(skb, &vlan_tci)) {
3908 				tx_start_bd->vlan_or_ethertype =
3909 					cpu_to_le16(ntohs(eth->h_proto));
3910 			} else {
3911 				tx_start_bd->bd_flags.as_bitfield |=
3912 					(X_ETH_INBAND_VLAN <<
3913 					 ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3914 				tx_start_bd->vlan_or_ethertype =
3915 					cpu_to_le16(vlan_tci);
3916 			}
3917 #ifndef BNX2X_STOP_ON_ERROR
3918 		} else {
3919 			/* used by FW for packet accounting */
3920 			tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3921 		}
3922 #endif
3923 	}
3924 
3925 	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3926 
3927 	/* turn on parsing and get a BD */
3928 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3929 
3930 	if (xmit_type & XMIT_CSUM)
3931 		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3932 
3933 	if (!CHIP_IS_E1x(bp)) {
3934 		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3935 		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3936 
3937 		if (xmit_type & XMIT_CSUM_ENC) {
3938 			u16 global_data = 0;
3939 
3940 			/* Set PBD in enc checksum offload case */
3941 			hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3942 						      &pbd_e2_parsing_data,
3943 						      xmit_type);
3944 
3945 			/* turn on 2nd parsing and get a BD */
3946 			bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3947 
3948 			pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3949 
3950 			memset(pbd2, 0, sizeof(*pbd2));
3951 
3952 			pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3953 				(skb_inner_network_header(skb) -
3954 				 skb->data) >> 1;
3955 
3956 			if (xmit_type & XMIT_GSO_ENC)
3957 				bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3958 							  &global_data,
3959 							  xmit_type);
3960 
3961 			pbd2->global_data = cpu_to_le16(global_data);
3962 
3963 			/* add addition parse BD indication to start BD */
3964 			SET_FLAG(tx_start_bd->general_data,
3965 				 ETH_TX_START_BD_PARSE_NBDS, 1);
3966 			/* set encapsulation flag in start BD */
3967 			SET_FLAG(tx_start_bd->general_data,
3968 				 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3969 
3970 			tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3971 
3972 			nbd++;
3973 		} else if (xmit_type & XMIT_CSUM) {
3974 			/* Set PBD in checksum offload case w/o encapsulation */
3975 			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3976 						     &pbd_e2_parsing_data,
3977 						     xmit_type);
3978 		}
3979 
3980 		bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3981 		/* Add the macs to the parsing BD if this is a vf or if
3982 		 * Tx Switching is enabled.
3983 		 */
3984 		if (IS_VF(bp)) {
3985 			/* override GRE parameters in BD */
3986 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3987 					      &pbd_e2->data.mac_addr.src_mid,
3988 					      &pbd_e2->data.mac_addr.src_lo,
3989 					      eth->h_source);
3990 
3991 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3992 					      &pbd_e2->data.mac_addr.dst_mid,
3993 					      &pbd_e2->data.mac_addr.dst_lo,
3994 					      eth->h_dest);
3995 		} else {
3996 			if (bp->flags & TX_SWITCHING)
3997 				bnx2x_set_fw_mac_addr(
3998 						&pbd_e2->data.mac_addr.dst_hi,
3999 						&pbd_e2->data.mac_addr.dst_mid,
4000 						&pbd_e2->data.mac_addr.dst_lo,
4001 						eth->h_dest);
4002 #ifdef BNX2X_STOP_ON_ERROR
4003 			/* Enforce security is always set in Stop on Error -
4004 			 * source mac should be present in the parsing BD
4005 			 */
4006 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
4007 					      &pbd_e2->data.mac_addr.src_mid,
4008 					      &pbd_e2->data.mac_addr.src_lo,
4009 					      eth->h_source);
4010 #endif
4011 		}
4012 
4013 		SET_FLAG(pbd_e2_parsing_data,
4014 			 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
4015 	} else {
4016 		u16 global_data = 0;
4017 		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4018 		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4019 		/* Set PBD in checksum offload case */
4020 		if (xmit_type & XMIT_CSUM)
4021 			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4022 
4023 		SET_FLAG(global_data,
4024 			 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4025 		pbd_e1x->global_data |= cpu_to_le16(global_data);
4026 	}
4027 
4028 	/* Setup the data pointer of the first BD of the packet */
4029 	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4030 	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4031 	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4032 	pkt_size = tx_start_bd->nbytes;
4033 
4034 	DP(NETIF_MSG_TX_QUEUED,
4035 	   "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4036 	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4037 	   le16_to_cpu(tx_start_bd->nbytes),
4038 	   tx_start_bd->bd_flags.as_bitfield,
4039 	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4040 
4041 	if (xmit_type & XMIT_GSO) {
4042 
4043 		DP(NETIF_MSG_TX_QUEUED,
4044 		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4045 		   skb->len, hlen, skb_headlen(skb),
4046 		   skb_shinfo(skb)->gso_size);
4047 
4048 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4049 
4050 		if (unlikely(skb_headlen(skb) > hlen)) {
4051 			nbd++;
4052 			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4053 						 &tx_start_bd, hlen,
4054 						 bd_prod);
4055 		}
4056 		if (!CHIP_IS_E1x(bp))
4057 			pbd_e2_parsing_data |=
4058 				(skb_shinfo(skb)->gso_size <<
4059 				 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4060 				 ETH_TX_PARSE_BD_E2_LSO_MSS;
4061 		else
4062 			bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4063 	}
4064 
4065 	/* Set the PBD's parsing_data field if not zero
4066 	 * (for the chips newer than 57711).
4067 	 */
4068 	if (pbd_e2_parsing_data)
4069 		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4070 
4071 	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4072 
4073 	/* Handle fragmented skb */
4074 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4075 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4076 
4077 		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4078 					   skb_frag_size(frag), DMA_TO_DEVICE);
4079 		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4080 			unsigned int pkts_compl = 0, bytes_compl = 0;
4081 
4082 			DP(NETIF_MSG_TX_QUEUED,
4083 			   "Unable to map page - dropping packet...\n");
4084 
4085 			/* we need unmap all buffers already mapped
4086 			 * for this SKB;
4087 			 * first_bd->nbd need to be properly updated
4088 			 * before call to bnx2x_free_tx_pkt
4089 			 */
4090 			first_bd->nbd = cpu_to_le16(nbd);
4091 			bnx2x_free_tx_pkt(bp, txdata,
4092 					  TX_BD(txdata->tx_pkt_prod),
4093 					  &pkts_compl, &bytes_compl);
4094 			return NETDEV_TX_OK;
4095 		}
4096 
4097 		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4098 		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4099 		if (total_pkt_bd == NULL)
4100 			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4101 
4102 		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4103 		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4104 		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4105 		le16_add_cpu(&pkt_size, skb_frag_size(frag));
4106 		nbd++;
4107 
4108 		DP(NETIF_MSG_TX_QUEUED,
4109 		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4110 		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4111 		   le16_to_cpu(tx_data_bd->nbytes));
4112 	}
4113 
4114 	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4115 
4116 	/* update with actual num BDs */
4117 	first_bd->nbd = cpu_to_le16(nbd);
4118 
4119 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4120 
4121 	/* now send a tx doorbell, counting the next BD
4122 	 * if the packet contains or ends with it
4123 	 */
4124 	if (TX_BD_POFF(bd_prod) < nbd)
4125 		nbd++;
4126 
4127 	/* total_pkt_bytes should be set on the first data BD if
4128 	 * it's not an LSO packet and there is more than one
4129 	 * data BD. In this case pkt_size is limited by an MTU value.
4130 	 * However we prefer to set it for an LSO packet (while we don't
4131 	 * have to) in order to save some CPU cycles in a none-LSO
4132 	 * case, when we much more care about them.
4133 	 */
4134 	if (total_pkt_bd != NULL)
4135 		total_pkt_bd->total_pkt_bytes = pkt_size;
4136 
4137 	if (pbd_e1x)
4138 		DP(NETIF_MSG_TX_QUEUED,
4139 		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4140 		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4141 		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4142 		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4143 		    le16_to_cpu(pbd_e1x->total_hlen_w));
4144 	if (pbd_e2)
4145 		DP(NETIF_MSG_TX_QUEUED,
4146 		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4147 		   pbd_e2,
4148 		   pbd_e2->data.mac_addr.dst_hi,
4149 		   pbd_e2->data.mac_addr.dst_mid,
4150 		   pbd_e2->data.mac_addr.dst_lo,
4151 		   pbd_e2->data.mac_addr.src_hi,
4152 		   pbd_e2->data.mac_addr.src_mid,
4153 		   pbd_e2->data.mac_addr.src_lo,
4154 		   pbd_e2->parsing_data);
4155 	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4156 
4157 	netdev_tx_sent_queue(txq, skb->len);
4158 
4159 	skb_tx_timestamp(skb);
4160 
4161 	txdata->tx_pkt_prod++;
4162 	/*
4163 	 * Make sure that the BD data is updated before updating the producer
4164 	 * since FW might read the BD right after the producer is updated.
4165 	 * This is only applicable for weak-ordered memory model archs such
4166 	 * as IA-64. The following barrier is also mandatory since FW will
4167 	 * assumes packets must have BDs.
4168 	 */
4169 	wmb();
4170 
4171 	txdata->tx_db.data.prod += nbd;
4172 	/* make sure descriptor update is observed by HW */
4173 	wmb();
4174 
4175 	DOORBELL_RELAXED(bp, txdata->cid, txdata->tx_db.raw);
4176 
4177 	txdata->tx_bd_prod += nbd;
4178 
4179 	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4180 		netif_tx_stop_queue(txq);
4181 
4182 		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
4183 		 * ordering of set_bit() in netif_tx_stop_queue() and read of
4184 		 * fp->bd_tx_cons */
4185 		smp_mb();
4186 
4187 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4188 		if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4189 			netif_tx_wake_queue(txq);
4190 	}
4191 	txdata->tx_pkt++;
4192 
4193 	return NETDEV_TX_OK;
4194 }
4195 
bnx2x_get_c2s_mapping(struct bnx2x * bp,u8 * c2s_map,u8 * c2s_default)4196 void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default)
4197 {
4198 	int mfw_vn = BP_FW_MB_IDX(bp);
4199 	u32 tmp;
4200 
4201 	/* If the shmem shouldn't affect configuration, reflect */
4202 	if (!IS_MF_BD(bp)) {
4203 		int i;
4204 
4205 		for (i = 0; i < BNX2X_MAX_PRIORITY; i++)
4206 			c2s_map[i] = i;
4207 		*c2s_default = 0;
4208 
4209 		return;
4210 	}
4211 
4212 	tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]);
4213 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4214 	c2s_map[0] = tmp & 0xff;
4215 	c2s_map[1] = (tmp >> 8) & 0xff;
4216 	c2s_map[2] = (tmp >> 16) & 0xff;
4217 	c2s_map[3] = (tmp >> 24) & 0xff;
4218 
4219 	tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]);
4220 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4221 	c2s_map[4] = tmp & 0xff;
4222 	c2s_map[5] = (tmp >> 8) & 0xff;
4223 	c2s_map[6] = (tmp >> 16) & 0xff;
4224 	c2s_map[7] = (tmp >> 24) & 0xff;
4225 
4226 	tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]);
4227 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4228 	*c2s_default = (tmp >> (8 * mfw_vn)) & 0xff;
4229 }
4230 
4231 /**
4232  * bnx2x_setup_tc - routine to configure net_device for multi tc
4233  *
4234  * @dev: net device to configure
4235  * @num_tc: number of traffic classes to enable
4236  *
4237  * callback connected to the ndo_setup_tc function pointer
4238  */
bnx2x_setup_tc(struct net_device * dev,u8 num_tc)4239 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4240 {
4241 	struct bnx2x *bp = netdev_priv(dev);
4242 	u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def;
4243 	int cos, prio, count, offset;
4244 
4245 	/* setup tc must be called under rtnl lock */
4246 	ASSERT_RTNL();
4247 
4248 	/* no traffic classes requested. Aborting */
4249 	if (!num_tc) {
4250 		netdev_reset_tc(dev);
4251 		return 0;
4252 	}
4253 
4254 	/* requested to support too many traffic classes */
4255 	if (num_tc > bp->max_cos) {
4256 		BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4257 			  num_tc, bp->max_cos);
4258 		return -EINVAL;
4259 	}
4260 
4261 	/* declare amount of supported traffic classes */
4262 	if (netdev_set_num_tc(dev, num_tc)) {
4263 		BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4264 		return -EINVAL;
4265 	}
4266 
4267 	bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def);
4268 
4269 	/* configure priority to traffic class mapping */
4270 	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4271 		int outer_prio = c2s_map[prio];
4272 
4273 		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]);
4274 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4275 		   "mapping priority %d to tc %d\n",
4276 		   outer_prio, bp->prio_to_cos[outer_prio]);
4277 	}
4278 
4279 	/* Use this configuration to differentiate tc0 from other COSes
4280 	   This can be used for ets or pfc, and save the effort of setting
4281 	   up a multio class queue disc or negotiating DCBX with a switch
4282 	netdev_set_prio_tc_map(dev, 0, 0);
4283 	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4284 	for (prio = 1; prio < 16; prio++) {
4285 		netdev_set_prio_tc_map(dev, prio, 1);
4286 		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4287 	} */
4288 
4289 	/* configure traffic class to transmission queue mapping */
4290 	for (cos = 0; cos < bp->max_cos; cos++) {
4291 		count = BNX2X_NUM_ETH_QUEUES(bp);
4292 		offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4293 		netdev_set_tc_queue(dev, cos, count, offset);
4294 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4295 		   "mapping tc %d to offset %d count %d\n",
4296 		   cos, offset, count);
4297 	}
4298 
4299 	return 0;
4300 }
4301 
__bnx2x_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)4302 int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
4303 		     void *type_data)
4304 {
4305 	struct tc_mqprio_qopt *mqprio = type_data;
4306 
4307 	if (type != TC_SETUP_QDISC_MQPRIO)
4308 		return -EOPNOTSUPP;
4309 
4310 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
4311 
4312 	return bnx2x_setup_tc(dev, mqprio->num_tc);
4313 }
4314 
4315 /* called with rtnl_lock */
bnx2x_change_mac_addr(struct net_device * dev,void * p)4316 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4317 {
4318 	struct sockaddr *addr = p;
4319 	struct bnx2x *bp = netdev_priv(dev);
4320 	int rc = 0;
4321 
4322 	if (!is_valid_ether_addr(addr->sa_data)) {
4323 		BNX2X_ERR("Requested MAC address is not valid\n");
4324 		return -EINVAL;
4325 	}
4326 
4327 	if (IS_MF_STORAGE_ONLY(bp)) {
4328 		BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4329 		return -EINVAL;
4330 	}
4331 
4332 	if (netif_running(dev))  {
4333 		rc = bnx2x_set_eth_mac(bp, false);
4334 		if (rc)
4335 			return rc;
4336 	}
4337 
4338 	eth_hw_addr_set(dev, addr->sa_data);
4339 
4340 	if (netif_running(dev))
4341 		rc = bnx2x_set_eth_mac(bp, true);
4342 
4343 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4344 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4345 
4346 	return rc;
4347 }
4348 
bnx2x_free_fp_mem_at(struct bnx2x * bp,int fp_index)4349 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4350 {
4351 	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4352 	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4353 	u8 cos;
4354 
4355 	/* Common */
4356 
4357 	if (IS_FCOE_IDX(fp_index)) {
4358 		memset(sb, 0, sizeof(union host_hc_status_block));
4359 		fp->status_blk_mapping = 0;
4360 	} else {
4361 		/* status blocks */
4362 		if (!CHIP_IS_E1x(bp))
4363 			BNX2X_PCI_FREE(sb->e2_sb,
4364 				       bnx2x_fp(bp, fp_index,
4365 						status_blk_mapping),
4366 				       sizeof(struct host_hc_status_block_e2));
4367 		else
4368 			BNX2X_PCI_FREE(sb->e1x_sb,
4369 				       bnx2x_fp(bp, fp_index,
4370 						status_blk_mapping),
4371 				       sizeof(struct host_hc_status_block_e1x));
4372 	}
4373 
4374 	/* Rx */
4375 	if (!skip_rx_queue(bp, fp_index)) {
4376 		bnx2x_free_rx_bds(fp);
4377 
4378 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4379 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4380 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4381 			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
4382 			       sizeof(struct eth_rx_bd) * NUM_RX_BD);
4383 
4384 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4385 			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
4386 			       sizeof(struct eth_fast_path_rx_cqe) *
4387 			       NUM_RCQ_BD);
4388 
4389 		/* SGE ring */
4390 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4391 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4392 			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
4393 			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4394 	}
4395 
4396 	/* Tx */
4397 	if (!skip_tx_queue(bp, fp_index)) {
4398 		/* fastpath tx rings: tx_buf tx_desc */
4399 		for_each_cos_in_tx_queue(fp, cos) {
4400 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4401 
4402 			DP(NETIF_MSG_IFDOWN,
4403 			   "freeing tx memory of fp %d cos %d cid %d\n",
4404 			   fp_index, cos, txdata->cid);
4405 
4406 			BNX2X_FREE(txdata->tx_buf_ring);
4407 			BNX2X_PCI_FREE(txdata->tx_desc_ring,
4408 				txdata->tx_desc_mapping,
4409 				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4410 		}
4411 	}
4412 	/* end of fastpath */
4413 }
4414 
bnx2x_free_fp_mem_cnic(struct bnx2x * bp)4415 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4416 {
4417 	int i;
4418 	for_each_cnic_queue(bp, i)
4419 		bnx2x_free_fp_mem_at(bp, i);
4420 }
4421 
bnx2x_free_fp_mem(struct bnx2x * bp)4422 void bnx2x_free_fp_mem(struct bnx2x *bp)
4423 {
4424 	int i;
4425 	for_each_eth_queue(bp, i)
4426 		bnx2x_free_fp_mem_at(bp, i);
4427 }
4428 
set_sb_shortcuts(struct bnx2x * bp,int index)4429 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4430 {
4431 	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4432 	if (!CHIP_IS_E1x(bp)) {
4433 		bnx2x_fp(bp, index, sb_index_values) =
4434 			(__le16 *)status_blk.e2_sb->sb.index_values;
4435 		bnx2x_fp(bp, index, sb_running_index) =
4436 			(__le16 *)status_blk.e2_sb->sb.running_index;
4437 	} else {
4438 		bnx2x_fp(bp, index, sb_index_values) =
4439 			(__le16 *)status_blk.e1x_sb->sb.index_values;
4440 		bnx2x_fp(bp, index, sb_running_index) =
4441 			(__le16 *)status_blk.e1x_sb->sb.running_index;
4442 	}
4443 }
4444 
4445 /* Returns the number of actually allocated BDs */
bnx2x_alloc_rx_bds(struct bnx2x_fastpath * fp,int rx_ring_size)4446 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4447 			      int rx_ring_size)
4448 {
4449 	struct bnx2x *bp = fp->bp;
4450 	u16 ring_prod, cqe_ring_prod;
4451 	int i, failure_cnt = 0;
4452 
4453 	fp->rx_comp_cons = 0;
4454 	cqe_ring_prod = ring_prod = 0;
4455 
4456 	/* This routine is called only during fo init so
4457 	 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4458 	 */
4459 	for (i = 0; i < rx_ring_size; i++) {
4460 		if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4461 			failure_cnt++;
4462 			continue;
4463 		}
4464 		ring_prod = NEXT_RX_IDX(ring_prod);
4465 		cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4466 		WARN_ON(ring_prod <= (i - failure_cnt));
4467 	}
4468 
4469 	if (failure_cnt)
4470 		BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4471 			  i - failure_cnt, fp->index);
4472 
4473 	fp->rx_bd_prod = ring_prod;
4474 	/* Limit the CQE producer by the CQE ring size */
4475 	fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4476 			       cqe_ring_prod);
4477 
4478 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4479 
4480 	return i - failure_cnt;
4481 }
4482 
bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath * fp)4483 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4484 {
4485 	int i;
4486 
4487 	for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4488 		struct eth_rx_cqe_next_page *nextpg;
4489 
4490 		nextpg = (struct eth_rx_cqe_next_page *)
4491 			&fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4492 		nextpg->addr_hi =
4493 			cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4494 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4495 		nextpg->addr_lo =
4496 			cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4497 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4498 	}
4499 }
4500 
bnx2x_alloc_fp_mem_at(struct bnx2x * bp,int index)4501 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4502 {
4503 	union host_hc_status_block *sb;
4504 	struct bnx2x_fastpath *fp = &bp->fp[index];
4505 	int ring_size = 0;
4506 	u8 cos;
4507 	int rx_ring_size = 0;
4508 
4509 	if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4510 		rx_ring_size = MIN_RX_SIZE_NONTPA;
4511 		bp->rx_ring_size = rx_ring_size;
4512 	} else if (!bp->rx_ring_size) {
4513 		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4514 
4515 		if (CHIP_IS_E3(bp)) {
4516 			u32 cfg = SHMEM_RD(bp,
4517 					   dev_info.port_hw_config[BP_PORT(bp)].
4518 					   default_cfg);
4519 
4520 			/* Decrease ring size for 1G functions */
4521 			if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4522 			    PORT_HW_CFG_NET_SERDES_IF_SGMII)
4523 				rx_ring_size /= 10;
4524 		}
4525 
4526 		/* allocate at least number of buffers required by FW */
4527 		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4528 				     MIN_RX_SIZE_TPA, rx_ring_size);
4529 
4530 		bp->rx_ring_size = rx_ring_size;
4531 	} else /* if rx_ring_size specified - use it */
4532 		rx_ring_size = bp->rx_ring_size;
4533 
4534 	DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4535 
4536 	/* Common */
4537 	sb = &bnx2x_fp(bp, index, status_blk);
4538 
4539 	if (!IS_FCOE_IDX(index)) {
4540 		/* status blocks */
4541 		if (!CHIP_IS_E1x(bp)) {
4542 			sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4543 						    sizeof(struct host_hc_status_block_e2));
4544 			if (!sb->e2_sb)
4545 				goto alloc_mem_err;
4546 		} else {
4547 			sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4548 						     sizeof(struct host_hc_status_block_e1x));
4549 			if (!sb->e1x_sb)
4550 				goto alloc_mem_err;
4551 		}
4552 	}
4553 
4554 	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4555 	 * set shortcuts for it.
4556 	 */
4557 	if (!IS_FCOE_IDX(index))
4558 		set_sb_shortcuts(bp, index);
4559 
4560 	/* Tx */
4561 	if (!skip_tx_queue(bp, index)) {
4562 		/* fastpath tx rings: tx_buf tx_desc */
4563 		for_each_cos_in_tx_queue(fp, cos) {
4564 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4565 
4566 			DP(NETIF_MSG_IFUP,
4567 			   "allocating tx memory of fp %d cos %d\n",
4568 			   index, cos);
4569 
4570 			txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4571 						      sizeof(struct sw_tx_bd),
4572 						      GFP_KERNEL);
4573 			if (!txdata->tx_buf_ring)
4574 				goto alloc_mem_err;
4575 			txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4576 							       sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4577 			if (!txdata->tx_desc_ring)
4578 				goto alloc_mem_err;
4579 		}
4580 	}
4581 
4582 	/* Rx */
4583 	if (!skip_rx_queue(bp, index)) {
4584 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4585 		bnx2x_fp(bp, index, rx_buf_ring) =
4586 			kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4587 		if (!bnx2x_fp(bp, index, rx_buf_ring))
4588 			goto alloc_mem_err;
4589 		bnx2x_fp(bp, index, rx_desc_ring) =
4590 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4591 					sizeof(struct eth_rx_bd) * NUM_RX_BD);
4592 		if (!bnx2x_fp(bp, index, rx_desc_ring))
4593 			goto alloc_mem_err;
4594 
4595 		/* Seed all CQEs by 1s */
4596 		bnx2x_fp(bp, index, rx_comp_ring) =
4597 			BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4598 					 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4599 		if (!bnx2x_fp(bp, index, rx_comp_ring))
4600 			goto alloc_mem_err;
4601 
4602 		/* SGE ring */
4603 		bnx2x_fp(bp, index, rx_page_ring) =
4604 			kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4605 				GFP_KERNEL);
4606 		if (!bnx2x_fp(bp, index, rx_page_ring))
4607 			goto alloc_mem_err;
4608 		bnx2x_fp(bp, index, rx_sge_ring) =
4609 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4610 					BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4611 		if (!bnx2x_fp(bp, index, rx_sge_ring))
4612 			goto alloc_mem_err;
4613 		/* RX BD ring */
4614 		bnx2x_set_next_page_rx_bd(fp);
4615 
4616 		/* CQ ring */
4617 		bnx2x_set_next_page_rx_cq(fp);
4618 
4619 		/* BDs */
4620 		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4621 		if (ring_size < rx_ring_size)
4622 			goto alloc_mem_err;
4623 	}
4624 
4625 	return 0;
4626 
4627 /* handles low memory cases */
4628 alloc_mem_err:
4629 	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4630 						index, ring_size);
4631 	/* FW will drop all packets if queue is not big enough,
4632 	 * In these cases we disable the queue
4633 	 * Min size is different for OOO, TPA and non-TPA queues
4634 	 */
4635 	if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4636 				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4637 			/* release memory allocated for this queue */
4638 			bnx2x_free_fp_mem_at(bp, index);
4639 			return -ENOMEM;
4640 	}
4641 	return 0;
4642 }
4643 
bnx2x_alloc_fp_mem_cnic(struct bnx2x * bp)4644 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4645 {
4646 	if (!NO_FCOE(bp))
4647 		/* FCoE */
4648 		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4649 			/* we will fail load process instead of mark
4650 			 * NO_FCOE_FLAG
4651 			 */
4652 			return -ENOMEM;
4653 
4654 	return 0;
4655 }
4656 
bnx2x_alloc_fp_mem(struct bnx2x * bp)4657 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4658 {
4659 	int i;
4660 
4661 	/* 1. Allocate FP for leading - fatal if error
4662 	 * 2. Allocate RSS - fix number of queues if error
4663 	 */
4664 
4665 	/* leading */
4666 	if (bnx2x_alloc_fp_mem_at(bp, 0))
4667 		return -ENOMEM;
4668 
4669 	/* RSS */
4670 	for_each_nondefault_eth_queue(bp, i)
4671 		if (bnx2x_alloc_fp_mem_at(bp, i))
4672 			break;
4673 
4674 	/* handle memory failures */
4675 	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4676 		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4677 
4678 		WARN_ON(delta < 0);
4679 		bnx2x_shrink_eth_fp(bp, delta);
4680 		if (CNIC_SUPPORT(bp))
4681 			/* move non eth FPs next to last eth FP
4682 			 * must be done in that order
4683 			 * FCOE_IDX < FWD_IDX < OOO_IDX
4684 			 */
4685 
4686 			/* move FCoE fp even NO_FCOE_FLAG is on */
4687 			bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4688 		bp->num_ethernet_queues -= delta;
4689 		bp->num_queues = bp->num_ethernet_queues +
4690 				 bp->num_cnic_queues;
4691 		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4692 			  bp->num_queues + delta, bp->num_queues);
4693 	}
4694 
4695 	return 0;
4696 }
4697 
bnx2x_free_mem_bp(struct bnx2x * bp)4698 void bnx2x_free_mem_bp(struct bnx2x *bp)
4699 {
4700 	int i;
4701 
4702 	for (i = 0; i < bp->fp_array_size; i++)
4703 		kfree(bp->fp[i].tpa_info);
4704 	kfree(bp->fp);
4705 	kfree(bp->sp_objs);
4706 	kfree(bp->fp_stats);
4707 	kfree(bp->bnx2x_txq);
4708 	kfree(bp->msix_table);
4709 	kfree(bp->ilt);
4710 }
4711 
bnx2x_alloc_mem_bp(struct bnx2x * bp)4712 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4713 {
4714 	struct bnx2x_fastpath *fp;
4715 	struct msix_entry *tbl;
4716 	struct bnx2x_ilt *ilt;
4717 	int msix_table_size = 0;
4718 	int fp_array_size, txq_array_size;
4719 	int i;
4720 
4721 	/*
4722 	 * The biggest MSI-X table we might need is as a maximum number of fast
4723 	 * path IGU SBs plus default SB (for PF only).
4724 	 */
4725 	msix_table_size = bp->igu_sb_cnt;
4726 	if (IS_PF(bp))
4727 		msix_table_size++;
4728 	BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4729 
4730 	/* fp array: RSS plus CNIC related L2 queues */
4731 	fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4732 	bp->fp_array_size = fp_array_size;
4733 	BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4734 
4735 	fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4736 	if (!fp)
4737 		goto alloc_err;
4738 	for (i = 0; i < bp->fp_array_size; i++) {
4739 		fp[i].tpa_info =
4740 			kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4741 				sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4742 		if (!(fp[i].tpa_info))
4743 			goto alloc_err;
4744 	}
4745 
4746 	bp->fp = fp;
4747 
4748 	/* allocate sp objs */
4749 	bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4750 			      GFP_KERNEL);
4751 	if (!bp->sp_objs)
4752 		goto alloc_err;
4753 
4754 	/* allocate fp_stats */
4755 	bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4756 			       GFP_KERNEL);
4757 	if (!bp->fp_stats)
4758 		goto alloc_err;
4759 
4760 	/* Allocate memory for the transmission queues array */
4761 	txq_array_size =
4762 		BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4763 	BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4764 
4765 	bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4766 				GFP_KERNEL);
4767 	if (!bp->bnx2x_txq)
4768 		goto alloc_err;
4769 
4770 	/* msix table */
4771 	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4772 	if (!tbl)
4773 		goto alloc_err;
4774 	bp->msix_table = tbl;
4775 
4776 	/* ilt */
4777 	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4778 	if (!ilt)
4779 		goto alloc_err;
4780 	bp->ilt = ilt;
4781 
4782 	return 0;
4783 alloc_err:
4784 	bnx2x_free_mem_bp(bp);
4785 	return -ENOMEM;
4786 }
4787 
bnx2x_reload_if_running(struct net_device * dev)4788 int bnx2x_reload_if_running(struct net_device *dev)
4789 {
4790 	struct bnx2x *bp = netdev_priv(dev);
4791 
4792 	if (unlikely(!netif_running(dev)))
4793 		return 0;
4794 
4795 	bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4796 	return bnx2x_nic_load(bp, LOAD_NORMAL);
4797 }
4798 
bnx2x_get_cur_phy_idx(struct bnx2x * bp)4799 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4800 {
4801 	u32 sel_phy_idx = 0;
4802 	if (bp->link_params.num_phys <= 1)
4803 		return INT_PHY;
4804 
4805 	if (bp->link_vars.link_up) {
4806 		sel_phy_idx = EXT_PHY1;
4807 		/* In case link is SERDES, check if the EXT_PHY2 is the one */
4808 		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4809 		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4810 			sel_phy_idx = EXT_PHY2;
4811 	} else {
4812 
4813 		switch (bnx2x_phy_selection(&bp->link_params)) {
4814 		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4815 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4816 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4817 		       sel_phy_idx = EXT_PHY1;
4818 		       break;
4819 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4820 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4821 		       sel_phy_idx = EXT_PHY2;
4822 		       break;
4823 		}
4824 	}
4825 
4826 	return sel_phy_idx;
4827 }
bnx2x_get_link_cfg_idx(struct bnx2x * bp)4828 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4829 {
4830 	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4831 	/*
4832 	 * The selected activated PHY is always after swapping (in case PHY
4833 	 * swapping is enabled). So when swapping is enabled, we need to reverse
4834 	 * the configuration
4835 	 */
4836 
4837 	if (bp->link_params.multi_phy_config &
4838 	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4839 		if (sel_phy_idx == EXT_PHY1)
4840 			sel_phy_idx = EXT_PHY2;
4841 		else if (sel_phy_idx == EXT_PHY2)
4842 			sel_phy_idx = EXT_PHY1;
4843 	}
4844 	return LINK_CONFIG_IDX(sel_phy_idx);
4845 }
4846 
4847 #ifdef NETDEV_FCOE_WWNN
bnx2x_fcoe_get_wwn(struct net_device * dev,u64 * wwn,int type)4848 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4849 {
4850 	struct bnx2x *bp = netdev_priv(dev);
4851 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4852 
4853 	switch (type) {
4854 	case NETDEV_FCOE_WWNN:
4855 		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4856 				cp->fcoe_wwn_node_name_lo);
4857 		break;
4858 	case NETDEV_FCOE_WWPN:
4859 		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4860 				cp->fcoe_wwn_port_name_lo);
4861 		break;
4862 	default:
4863 		BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4864 		return -EINVAL;
4865 	}
4866 
4867 	return 0;
4868 }
4869 #endif
4870 
4871 /* called with rtnl_lock */
bnx2x_change_mtu(struct net_device * dev,int new_mtu)4872 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4873 {
4874 	struct bnx2x *bp = netdev_priv(dev);
4875 
4876 	if (pci_num_vf(bp->pdev)) {
4877 		DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4878 		return -EPERM;
4879 	}
4880 
4881 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4882 		BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4883 		return -EAGAIN;
4884 	}
4885 
4886 	/* This does not race with packet allocation
4887 	 * because the actual alloc size is
4888 	 * only updated as part of load
4889 	 */
4890 	dev->mtu = new_mtu;
4891 
4892 	if (!bnx2x_mtu_allows_gro(new_mtu))
4893 		dev->features &= ~NETIF_F_GRO_HW;
4894 
4895 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4896 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4897 
4898 	return bnx2x_reload_if_running(dev);
4899 }
4900 
bnx2x_fix_features(struct net_device * dev,netdev_features_t features)4901 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4902 				     netdev_features_t features)
4903 {
4904 	struct bnx2x *bp = netdev_priv(dev);
4905 
4906 	if (pci_num_vf(bp->pdev)) {
4907 		netdev_features_t changed = dev->features ^ features;
4908 
4909 		/* Revert the requested changes in features if they
4910 		 * would require internal reload of PF in bnx2x_set_features().
4911 		 */
4912 		if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4913 			features &= ~NETIF_F_RXCSUM;
4914 			features |= dev->features & NETIF_F_RXCSUM;
4915 		}
4916 
4917 		if (changed & NETIF_F_LOOPBACK) {
4918 			features &= ~NETIF_F_LOOPBACK;
4919 			features |= dev->features & NETIF_F_LOOPBACK;
4920 		}
4921 	}
4922 
4923 	/* TPA requires Rx CSUM offloading */
4924 	if (!(features & NETIF_F_RXCSUM))
4925 		features &= ~NETIF_F_LRO;
4926 
4927 	if (!(features & NETIF_F_GRO) || !bnx2x_mtu_allows_gro(dev->mtu))
4928 		features &= ~NETIF_F_GRO_HW;
4929 	if (features & NETIF_F_GRO_HW)
4930 		features &= ~NETIF_F_LRO;
4931 
4932 	return features;
4933 }
4934 
bnx2x_set_features(struct net_device * dev,netdev_features_t features)4935 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4936 {
4937 	struct bnx2x *bp = netdev_priv(dev);
4938 	netdev_features_t changes = features ^ dev->features;
4939 	bool bnx2x_reload = false;
4940 	int rc;
4941 
4942 	/* VFs or non SRIOV PFs should be able to change loopback feature */
4943 	if (!pci_num_vf(bp->pdev)) {
4944 		if (features & NETIF_F_LOOPBACK) {
4945 			if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4946 				bp->link_params.loopback_mode = LOOPBACK_BMAC;
4947 				bnx2x_reload = true;
4948 			}
4949 		} else {
4950 			if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4951 				bp->link_params.loopback_mode = LOOPBACK_NONE;
4952 				bnx2x_reload = true;
4953 			}
4954 		}
4955 	}
4956 
4957 	/* Don't care about GRO changes */
4958 	changes &= ~NETIF_F_GRO;
4959 
4960 	if (changes)
4961 		bnx2x_reload = true;
4962 
4963 	if (bnx2x_reload) {
4964 		if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4965 			dev->features = features;
4966 			rc = bnx2x_reload_if_running(dev);
4967 			return rc ? rc : 1;
4968 		}
4969 		/* else: bnx2x_nic_load() will be called at end of recovery */
4970 	}
4971 
4972 	return 0;
4973 }
4974 
bnx2x_tx_timeout(struct net_device * dev,unsigned int txqueue)4975 void bnx2x_tx_timeout(struct net_device *dev, unsigned int txqueue)
4976 {
4977 	struct bnx2x *bp = netdev_priv(dev);
4978 
4979 	/* We want the information of the dump logged,
4980 	 * but calling bnx2x_panic() would kill all chances of recovery.
4981 	 */
4982 	if (!bp->panic)
4983 #ifndef BNX2X_STOP_ON_ERROR
4984 		bnx2x_panic_dump(bp, false);
4985 #else
4986 		bnx2x_panic();
4987 #endif
4988 
4989 	/* This allows the netif to be shutdown gracefully before resetting */
4990 	bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4991 }
4992 
bnx2x_suspend(struct device * dev_d)4993 static int __maybe_unused bnx2x_suspend(struct device *dev_d)
4994 {
4995 	struct pci_dev *pdev = to_pci_dev(dev_d);
4996 	struct net_device *dev = pci_get_drvdata(pdev);
4997 	struct bnx2x *bp;
4998 
4999 	if (!dev) {
5000 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5001 		return -ENODEV;
5002 	}
5003 	bp = netdev_priv(dev);
5004 
5005 	rtnl_lock();
5006 
5007 	if (!netif_running(dev)) {
5008 		rtnl_unlock();
5009 		return 0;
5010 	}
5011 
5012 	netif_device_detach(dev);
5013 
5014 	bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
5015 
5016 	rtnl_unlock();
5017 
5018 	return 0;
5019 }
5020 
bnx2x_resume(struct device * dev_d)5021 static int __maybe_unused bnx2x_resume(struct device *dev_d)
5022 {
5023 	struct pci_dev *pdev = to_pci_dev(dev_d);
5024 	struct net_device *dev = pci_get_drvdata(pdev);
5025 	struct bnx2x *bp;
5026 	int rc;
5027 
5028 	if (!dev) {
5029 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5030 		return -ENODEV;
5031 	}
5032 	bp = netdev_priv(dev);
5033 
5034 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
5035 		BNX2X_ERR("Handling parity error recovery. Try again later\n");
5036 		return -EAGAIN;
5037 	}
5038 
5039 	rtnl_lock();
5040 
5041 	if (!netif_running(dev)) {
5042 		rtnl_unlock();
5043 		return 0;
5044 	}
5045 
5046 	netif_device_attach(dev);
5047 
5048 	rc = bnx2x_nic_load(bp, LOAD_OPEN);
5049 
5050 	rtnl_unlock();
5051 
5052 	return rc;
5053 }
5054 
5055 SIMPLE_DEV_PM_OPS(bnx2x_pm_ops, bnx2x_suspend, bnx2x_resume);
5056 
bnx2x_set_ctx_validation(struct bnx2x * bp,struct eth_context * cxt,u32 cid)5057 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
5058 			      u32 cid)
5059 {
5060 	if (!cxt) {
5061 		BNX2X_ERR("bad context pointer %p\n", cxt);
5062 		return;
5063 	}
5064 
5065 	/* ustorm cxt validation */
5066 	cxt->ustorm_ag_context.cdu_usage =
5067 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5068 			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5069 	/* xcontext validation */
5070 	cxt->xstorm_ag_context.cdu_reserved =
5071 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5072 			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5073 }
5074 
storm_memset_hc_timeout(struct bnx2x * bp,u8 port,u8 fw_sb_id,u8 sb_index,u8 ticks)5075 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5076 				    u8 fw_sb_id, u8 sb_index,
5077 				    u8 ticks)
5078 {
5079 	u32 addr = BAR_CSTRORM_INTMEM +
5080 		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5081 	REG_WR8(bp, addr, ticks);
5082 	DP(NETIF_MSG_IFUP,
5083 	   "port %x fw_sb_id %d sb_index %d ticks %d\n",
5084 	   port, fw_sb_id, sb_index, ticks);
5085 }
5086 
storm_memset_hc_disable(struct bnx2x * bp,u8 port,u16 fw_sb_id,u8 sb_index,u8 disable)5087 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5088 				    u16 fw_sb_id, u8 sb_index,
5089 				    u8 disable)
5090 {
5091 	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5092 	u32 addr = BAR_CSTRORM_INTMEM +
5093 		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5094 	u8 flags = REG_RD8(bp, addr);
5095 	/* clear and set */
5096 	flags &= ~HC_INDEX_DATA_HC_ENABLED;
5097 	flags |= enable_flag;
5098 	REG_WR8(bp, addr, flags);
5099 	DP(NETIF_MSG_IFUP,
5100 	   "port %x fw_sb_id %d sb_index %d disable %d\n",
5101 	   port, fw_sb_id, sb_index, disable);
5102 }
5103 
bnx2x_update_coalesce_sb_index(struct bnx2x * bp,u8 fw_sb_id,u8 sb_index,u8 disable,u16 usec)5104 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5105 				    u8 sb_index, u8 disable, u16 usec)
5106 {
5107 	int port = BP_PORT(bp);
5108 	u8 ticks = usec / BNX2X_BTR;
5109 
5110 	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5111 
5112 	disable = disable ? 1 : (usec ? 0 : 1);
5113 	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5114 }
5115 
bnx2x_schedule_sp_rtnl(struct bnx2x * bp,enum sp_rtnl_flag flag,u32 verbose)5116 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5117 			    u32 verbose)
5118 {
5119 	smp_mb__before_atomic();
5120 	set_bit(flag, &bp->sp_rtnl_state);
5121 	smp_mb__after_atomic();
5122 	DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5123 	   flag);
5124 	schedule_delayed_work(&bp->sp_rtnl_task, 0);
5125 }
5126