1 /* bnx2x_cmn.c: QLogic Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  * Copyright (c) 2014 QLogic Corporation
5  * All rights reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation.
10  *
11  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
12  * Written by: Eliezer Tamir
13  * Based on code from Michael Chan's bnx2 driver
14  * UDP CSUM errata workaround by Arik Gendelman
15  * Slowpath and fastpath rework by Vladislav Zolotarov
16  * Statistics and Link management by Yitchak Gertner
17  *
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/etherdevice.h>
23 #include <linux/if_vlan.h>
24 #include <linux/interrupt.h>
25 #include <linux/ip.h>
26 #include <linux/crash_dump.h>
27 #include <net/tcp.h>
28 #include <net/gro.h>
29 #include <net/ipv6.h>
30 #include <net/ip6_checksum.h>
31 #include <linux/prefetch.h>
32 #include "bnx2x_cmn.h"
33 #include "bnx2x_init.h"
34 #include "bnx2x_sp.h"
35 
36 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
37 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
38 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
39 static int bnx2x_poll(struct napi_struct *napi, int budget);
40 
bnx2x_add_all_napi_cnic(struct bnx2x * bp)41 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
42 {
43 	int i;
44 
45 	/* Add NAPI objects */
46 	for_each_rx_queue_cnic(bp, i) {
47 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
48 			       bnx2x_poll, NAPI_POLL_WEIGHT);
49 	}
50 }
51 
bnx2x_add_all_napi(struct bnx2x * bp)52 static void bnx2x_add_all_napi(struct bnx2x *bp)
53 {
54 	int i;
55 
56 	/* Add NAPI objects */
57 	for_each_eth_queue(bp, i) {
58 		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
59 			       bnx2x_poll, NAPI_POLL_WEIGHT);
60 	}
61 }
62 
bnx2x_calc_num_queues(struct bnx2x * bp)63 static int bnx2x_calc_num_queues(struct bnx2x *bp)
64 {
65 	int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
66 
67 	/* Reduce memory usage in kdump environment by using only one queue */
68 	if (is_kdump_kernel())
69 		nq = 1;
70 
71 	nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
72 	return nq;
73 }
74 
75 /**
76  * bnx2x_move_fp - move content of the fastpath structure.
77  *
78  * @bp:		driver handle
79  * @from:	source FP index
80  * @to:		destination FP index
81  *
82  * Makes sure the contents of the bp->fp[to].napi is kept
83  * intact. This is done by first copying the napi struct from
84  * the target to the source, and then mem copying the entire
85  * source onto the target. Update txdata pointers and related
86  * content.
87  */
bnx2x_move_fp(struct bnx2x * bp,int from,int to)88 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
89 {
90 	struct bnx2x_fastpath *from_fp = &bp->fp[from];
91 	struct bnx2x_fastpath *to_fp = &bp->fp[to];
92 	struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
93 	struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
94 	struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
95 	struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
96 	int old_max_eth_txqs, new_max_eth_txqs;
97 	int old_txdata_index = 0, new_txdata_index = 0;
98 	struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
99 
100 	/* Copy the NAPI object as it has been already initialized */
101 	from_fp->napi = to_fp->napi;
102 
103 	/* Move bnx2x_fastpath contents */
104 	memcpy(to_fp, from_fp, sizeof(*to_fp));
105 	to_fp->index = to;
106 
107 	/* Retain the tpa_info of the original `to' version as we don't want
108 	 * 2 FPs to contain the same tpa_info pointer.
109 	 */
110 	to_fp->tpa_info = old_tpa_info;
111 
112 	/* move sp_objs contents as well, as their indices match fp ones */
113 	memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
114 
115 	/* move fp_stats contents as well, as their indices match fp ones */
116 	memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
117 
118 	/* Update txdata pointers in fp and move txdata content accordingly:
119 	 * Each fp consumes 'max_cos' txdata structures, so the index should be
120 	 * decremented by max_cos x delta.
121 	 */
122 
123 	old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
124 	new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
125 				(bp)->max_cos;
126 	if (from == FCOE_IDX(bp)) {
127 		old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
128 		new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
129 	}
130 
131 	memcpy(&bp->bnx2x_txq[new_txdata_index],
132 	       &bp->bnx2x_txq[old_txdata_index],
133 	       sizeof(struct bnx2x_fp_txdata));
134 	to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
135 }
136 
137 /**
138  * bnx2x_fill_fw_str - Fill buffer with FW version string.
139  *
140  * @bp:        driver handle
141  * @buf:       character buffer to fill with the fw name
142  * @buf_len:   length of the above buffer
143  *
144  */
bnx2x_fill_fw_str(struct bnx2x * bp,char * buf,size_t buf_len)145 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
146 {
147 	if (IS_PF(bp)) {
148 		u8 phy_fw_ver[PHY_FW_VER_LEN];
149 
150 		phy_fw_ver[0] = '\0';
151 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
152 					     phy_fw_ver, PHY_FW_VER_LEN);
153 		strlcpy(buf, bp->fw_ver, buf_len);
154 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
155 			 "bc %d.%d.%d%s%s",
156 			 (bp->common.bc_ver & 0xff0000) >> 16,
157 			 (bp->common.bc_ver & 0xff00) >> 8,
158 			 (bp->common.bc_ver & 0xff),
159 			 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
160 	} else {
161 		bnx2x_vf_fill_fw_str(bp, buf, buf_len);
162 	}
163 }
164 
165 /**
166  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
167  *
168  * @bp:	driver handle
169  * @delta:	number of eth queues which were not allocated
170  */
bnx2x_shrink_eth_fp(struct bnx2x * bp,int delta)171 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
172 {
173 	int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
174 
175 	/* Queue pointer cannot be re-set on an fp-basis, as moving pointer
176 	 * backward along the array could cause memory to be overridden
177 	 */
178 	for (cos = 1; cos < bp->max_cos; cos++) {
179 		for (i = 0; i < old_eth_num - delta; i++) {
180 			struct bnx2x_fastpath *fp = &bp->fp[i];
181 			int new_idx = cos * (old_eth_num - delta) + i;
182 
183 			memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
184 			       sizeof(struct bnx2x_fp_txdata));
185 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
186 		}
187 	}
188 }
189 
190 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
191 
192 /* free skb in the packet ring at pos idx
193  * return idx of last bd freed
194  */
bnx2x_free_tx_pkt(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata,u16 idx,unsigned int * pkts_compl,unsigned int * bytes_compl)195 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
196 			     u16 idx, unsigned int *pkts_compl,
197 			     unsigned int *bytes_compl)
198 {
199 	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
200 	struct eth_tx_start_bd *tx_start_bd;
201 	struct eth_tx_bd *tx_data_bd;
202 	struct sk_buff *skb = tx_buf->skb;
203 	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
204 	int nbd;
205 	u16 split_bd_len = 0;
206 
207 	/* prefetch skb end pointer to speedup dev_kfree_skb() */
208 	prefetch(&skb->end);
209 
210 	DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
211 	   txdata->txq_index, idx, tx_buf, skb);
212 
213 	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
214 
215 	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
216 #ifdef BNX2X_STOP_ON_ERROR
217 	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
218 		BNX2X_ERR("BAD nbd!\n");
219 		bnx2x_panic();
220 	}
221 #endif
222 	new_cons = nbd + tx_buf->first_bd;
223 
224 	/* Get the next bd */
225 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
226 
227 	/* Skip a parse bd... */
228 	--nbd;
229 	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
230 
231 	if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
232 		/* Skip second parse bd... */
233 		--nbd;
234 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
235 	}
236 
237 	/* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
238 	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
239 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
240 		split_bd_len = BD_UNMAP_LEN(tx_data_bd);
241 		--nbd;
242 		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
243 	}
244 
245 	/* unmap first bd */
246 	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
247 			 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
248 			 DMA_TO_DEVICE);
249 
250 	/* now free frags */
251 	while (nbd > 0) {
252 
253 		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
254 		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
255 			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
256 		if (--nbd)
257 			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
258 	}
259 
260 	/* release skb */
261 	WARN_ON(!skb);
262 	if (likely(skb)) {
263 		(*pkts_compl)++;
264 		(*bytes_compl) += skb->len;
265 		dev_kfree_skb_any(skb);
266 	}
267 
268 	tx_buf->first_bd = 0;
269 	tx_buf->skb = NULL;
270 
271 	return new_cons;
272 }
273 
bnx2x_tx_int(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata)274 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
275 {
276 	struct netdev_queue *txq;
277 	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
278 	unsigned int pkts_compl = 0, bytes_compl = 0;
279 
280 #ifdef BNX2X_STOP_ON_ERROR
281 	if (unlikely(bp->panic))
282 		return -1;
283 #endif
284 
285 	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
286 	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
287 	sw_cons = txdata->tx_pkt_cons;
288 
289 	/* Ensure subsequent loads occur after hw_cons */
290 	smp_rmb();
291 
292 	while (sw_cons != hw_cons) {
293 		u16 pkt_cons;
294 
295 		pkt_cons = TX_BD(sw_cons);
296 
297 		DP(NETIF_MSG_TX_DONE,
298 		   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
299 		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
300 
301 		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
302 					    &pkts_compl, &bytes_compl);
303 
304 		sw_cons++;
305 	}
306 
307 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
308 
309 	txdata->tx_pkt_cons = sw_cons;
310 	txdata->tx_bd_cons = bd_cons;
311 
312 	/* Need to make the tx_bd_cons update visible to start_xmit()
313 	 * before checking for netif_tx_queue_stopped().  Without the
314 	 * memory barrier, there is a small possibility that
315 	 * start_xmit() will miss it and cause the queue to be stopped
316 	 * forever.
317 	 * On the other hand we need an rmb() here to ensure the proper
318 	 * ordering of bit testing in the following
319 	 * netif_tx_queue_stopped(txq) call.
320 	 */
321 	smp_mb();
322 
323 	if (unlikely(netif_tx_queue_stopped(txq))) {
324 		/* Taking tx_lock() is needed to prevent re-enabling the queue
325 		 * while it's empty. This could have happen if rx_action() gets
326 		 * suspended in bnx2x_tx_int() after the condition before
327 		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
328 		 *
329 		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
330 		 * sends some packets consuming the whole queue again->
331 		 * stops the queue
332 		 */
333 
334 		__netif_tx_lock(txq, smp_processor_id());
335 
336 		if ((netif_tx_queue_stopped(txq)) &&
337 		    (bp->state == BNX2X_STATE_OPEN) &&
338 		    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
339 			netif_tx_wake_queue(txq);
340 
341 		__netif_tx_unlock(txq);
342 	}
343 	return 0;
344 }
345 
bnx2x_update_last_max_sge(struct bnx2x_fastpath * fp,u16 idx)346 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
347 					     u16 idx)
348 {
349 	u16 last_max = fp->last_max_sge;
350 
351 	if (SUB_S16(idx, last_max) > 0)
352 		fp->last_max_sge = idx;
353 }
354 
bnx2x_update_sge_prod(struct bnx2x_fastpath * fp,u16 sge_len,struct eth_end_agg_rx_cqe * cqe)355 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
356 					 u16 sge_len,
357 					 struct eth_end_agg_rx_cqe *cqe)
358 {
359 	struct bnx2x *bp = fp->bp;
360 	u16 last_max, last_elem, first_elem;
361 	u16 delta = 0;
362 	u16 i;
363 
364 	if (!sge_len)
365 		return;
366 
367 	/* First mark all used pages */
368 	for (i = 0; i < sge_len; i++)
369 		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
370 			RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
371 
372 	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
373 	   sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
374 
375 	/* Here we assume that the last SGE index is the biggest */
376 	prefetch((void *)(fp->sge_mask));
377 	bnx2x_update_last_max_sge(fp,
378 		le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
379 
380 	last_max = RX_SGE(fp->last_max_sge);
381 	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
382 	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
383 
384 	/* If ring is not full */
385 	if (last_elem + 1 != first_elem)
386 		last_elem++;
387 
388 	/* Now update the prod */
389 	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
390 		if (likely(fp->sge_mask[i]))
391 			break;
392 
393 		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
394 		delta += BIT_VEC64_ELEM_SZ;
395 	}
396 
397 	if (delta > 0) {
398 		fp->rx_sge_prod += delta;
399 		/* clear page-end entries */
400 		bnx2x_clear_sge_mask_next_elems(fp);
401 	}
402 
403 	DP(NETIF_MSG_RX_STATUS,
404 	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
405 	   fp->last_max_sge, fp->rx_sge_prod);
406 }
407 
408 /* Get Toeplitz hash value in the skb using the value from the
409  * CQE (calculated by HW).
410  */
bnx2x_get_rxhash(const struct bnx2x * bp,const struct eth_fast_path_rx_cqe * cqe,enum pkt_hash_types * rxhash_type)411 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
412 			    const struct eth_fast_path_rx_cqe *cqe,
413 			    enum pkt_hash_types *rxhash_type)
414 {
415 	/* Get Toeplitz hash from CQE */
416 	if ((bp->dev->features & NETIF_F_RXHASH) &&
417 	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
418 		enum eth_rss_hash_type htype;
419 
420 		htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
421 		*rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
422 				(htype == TCP_IPV6_HASH_TYPE)) ?
423 			       PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
424 
425 		return le32_to_cpu(cqe->rss_hash_result);
426 	}
427 	*rxhash_type = PKT_HASH_TYPE_NONE;
428 	return 0;
429 }
430 
bnx2x_tpa_start(struct bnx2x_fastpath * fp,u16 queue,u16 cons,u16 prod,struct eth_fast_path_rx_cqe * cqe)431 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
432 			    u16 cons, u16 prod,
433 			    struct eth_fast_path_rx_cqe *cqe)
434 {
435 	struct bnx2x *bp = fp->bp;
436 	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
437 	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
438 	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
439 	dma_addr_t mapping;
440 	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
441 	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
442 
443 	/* print error if current state != stop */
444 	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
445 		BNX2X_ERR("start of bin not in stop [%d]\n", queue);
446 
447 	/* Try to map an empty data buffer from the aggregation info  */
448 	mapping = dma_map_single(&bp->pdev->dev,
449 				 first_buf->data + NET_SKB_PAD,
450 				 fp->rx_buf_size, DMA_FROM_DEVICE);
451 	/*
452 	 *  ...if it fails - move the skb from the consumer to the producer
453 	 *  and set the current aggregation state as ERROR to drop it
454 	 *  when TPA_STOP arrives.
455 	 */
456 
457 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
458 		/* Move the BD from the consumer to the producer */
459 		bnx2x_reuse_rx_data(fp, cons, prod);
460 		tpa_info->tpa_state = BNX2X_TPA_ERROR;
461 		return;
462 	}
463 
464 	/* move empty data from pool to prod */
465 	prod_rx_buf->data = first_buf->data;
466 	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
467 	/* point prod_bd to new data */
468 	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
469 	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
470 
471 	/* move partial skb from cons to pool (don't unmap yet) */
472 	*first_buf = *cons_rx_buf;
473 
474 	/* mark bin state as START */
475 	tpa_info->parsing_flags =
476 		le16_to_cpu(cqe->pars_flags.flags);
477 	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
478 	tpa_info->tpa_state = BNX2X_TPA_START;
479 	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
480 	tpa_info->placement_offset = cqe->placement_offset;
481 	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
482 	if (fp->mode == TPA_MODE_GRO) {
483 		u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
484 		tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
485 		tpa_info->gro_size = gro_size;
486 	}
487 
488 #ifdef BNX2X_STOP_ON_ERROR
489 	fp->tpa_queue_used |= (1 << queue);
490 	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
491 	   fp->tpa_queue_used);
492 #endif
493 }
494 
495 /* Timestamp option length allowed for TPA aggregation:
496  *
497  *		nop nop kind length echo val
498  */
499 #define TPA_TSTAMP_OPT_LEN	12
500 /**
501  * bnx2x_set_gro_params - compute GRO values
502  *
503  * @skb:		packet skb
504  * @parsing_flags:	parsing flags from the START CQE
505  * @len_on_bd:		total length of the first packet for the
506  *			aggregation.
507  * @pkt_len:		length of all segments
508  * @num_of_coalesced_segs: count of segments
509  *
510  * Approximate value of the MSS for this aggregation calculated using
511  * the first packet of it.
512  * Compute number of aggregated segments, and gso_type.
513  */
bnx2x_set_gro_params(struct sk_buff * skb,u16 parsing_flags,u16 len_on_bd,unsigned int pkt_len,u16 num_of_coalesced_segs)514 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
515 				 u16 len_on_bd, unsigned int pkt_len,
516 				 u16 num_of_coalesced_segs)
517 {
518 	/* TPA aggregation won't have either IP options or TCP options
519 	 * other than timestamp or IPv6 extension headers.
520 	 */
521 	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
522 
523 	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
524 	    PRS_FLAG_OVERETH_IPV6) {
525 		hdrs_len += sizeof(struct ipv6hdr);
526 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
527 	} else {
528 		hdrs_len += sizeof(struct iphdr);
529 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
530 	}
531 
532 	/* Check if there was a TCP timestamp, if there is it's will
533 	 * always be 12 bytes length: nop nop kind length echo val.
534 	 *
535 	 * Otherwise FW would close the aggregation.
536 	 */
537 	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
538 		hdrs_len += TPA_TSTAMP_OPT_LEN;
539 
540 	skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
541 
542 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
543 	 * to skb_shinfo(skb)->gso_segs
544 	 */
545 	NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
546 }
547 
bnx2x_alloc_rx_sge(struct bnx2x * bp,struct bnx2x_fastpath * fp,u16 index,gfp_t gfp_mask)548 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
549 			      u16 index, gfp_t gfp_mask)
550 {
551 	struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
552 	struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
553 	struct bnx2x_alloc_pool *pool = &fp->page_pool;
554 	dma_addr_t mapping;
555 
556 	if (!pool->page) {
557 		pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
558 		if (unlikely(!pool->page))
559 			return -ENOMEM;
560 
561 		pool->offset = 0;
562 	}
563 
564 	mapping = dma_map_page(&bp->pdev->dev, pool->page,
565 			       pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE);
566 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
567 		BNX2X_ERR("Can't map sge\n");
568 		return -ENOMEM;
569 	}
570 
571 	sw_buf->page = pool->page;
572 	sw_buf->offset = pool->offset;
573 
574 	dma_unmap_addr_set(sw_buf, mapping, mapping);
575 
576 	sge->addr_hi = cpu_to_le32(U64_HI(mapping));
577 	sge->addr_lo = cpu_to_le32(U64_LO(mapping));
578 
579 	pool->offset += SGE_PAGE_SIZE;
580 	if (PAGE_SIZE - pool->offset >= SGE_PAGE_SIZE)
581 		get_page(pool->page);
582 	else
583 		pool->page = NULL;
584 	return 0;
585 }
586 
bnx2x_fill_frag_skb(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct bnx2x_agg_info * tpa_info,u16 pages,struct sk_buff * skb,struct eth_end_agg_rx_cqe * cqe,u16 cqe_idx)587 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
588 			       struct bnx2x_agg_info *tpa_info,
589 			       u16 pages,
590 			       struct sk_buff *skb,
591 			       struct eth_end_agg_rx_cqe *cqe,
592 			       u16 cqe_idx)
593 {
594 	struct sw_rx_page *rx_pg, old_rx_pg;
595 	u32 i, frag_len, frag_size;
596 	int err, j, frag_id = 0;
597 	u16 len_on_bd = tpa_info->len_on_bd;
598 	u16 full_page = 0, gro_size = 0;
599 
600 	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
601 
602 	if (fp->mode == TPA_MODE_GRO) {
603 		gro_size = tpa_info->gro_size;
604 		full_page = tpa_info->full_page;
605 	}
606 
607 	/* This is needed in order to enable forwarding support */
608 	if (frag_size)
609 		bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
610 				     le16_to_cpu(cqe->pkt_len),
611 				     le16_to_cpu(cqe->num_of_coalesced_segs));
612 
613 #ifdef BNX2X_STOP_ON_ERROR
614 	if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
615 		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
616 			  pages, cqe_idx);
617 		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
618 		bnx2x_panic();
619 		return -EINVAL;
620 	}
621 #endif
622 
623 	/* Run through the SGL and compose the fragmented skb */
624 	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
625 		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
626 
627 		/* FW gives the indices of the SGE as if the ring is an array
628 		   (meaning that "next" element will consume 2 indices) */
629 		if (fp->mode == TPA_MODE_GRO)
630 			frag_len = min_t(u32, frag_size, (u32)full_page);
631 		else /* LRO */
632 			frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
633 
634 		rx_pg = &fp->rx_page_ring[sge_idx];
635 		old_rx_pg = *rx_pg;
636 
637 		/* If we fail to allocate a substitute page, we simply stop
638 		   where we are and drop the whole packet */
639 		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
640 		if (unlikely(err)) {
641 			bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
642 			return err;
643 		}
644 
645 		dma_unmap_page(&bp->pdev->dev,
646 			       dma_unmap_addr(&old_rx_pg, mapping),
647 			       SGE_PAGE_SIZE, DMA_FROM_DEVICE);
648 		/* Add one frag and update the appropriate fields in the skb */
649 		if (fp->mode == TPA_MODE_LRO)
650 			skb_fill_page_desc(skb, j, old_rx_pg.page,
651 					   old_rx_pg.offset, frag_len);
652 		else { /* GRO */
653 			int rem;
654 			int offset = 0;
655 			for (rem = frag_len; rem > 0; rem -= gro_size) {
656 				int len = rem > gro_size ? gro_size : rem;
657 				skb_fill_page_desc(skb, frag_id++,
658 						   old_rx_pg.page,
659 						   old_rx_pg.offset + offset,
660 						   len);
661 				if (offset)
662 					get_page(old_rx_pg.page);
663 				offset += len;
664 			}
665 		}
666 
667 		skb->data_len += frag_len;
668 		skb->truesize += SGE_PAGES;
669 		skb->len += frag_len;
670 
671 		frag_size -= frag_len;
672 	}
673 
674 	return 0;
675 }
676 
bnx2x_frag_free(const struct bnx2x_fastpath * fp,void * data)677 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
678 {
679 	if (fp->rx_frag_size)
680 		skb_free_frag(data);
681 	else
682 		kfree(data);
683 }
684 
bnx2x_frag_alloc(const struct bnx2x_fastpath * fp,gfp_t gfp_mask)685 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
686 {
687 	if (fp->rx_frag_size) {
688 		/* GFP_KERNEL allocations are used only during initialization */
689 		if (unlikely(gfpflags_allow_blocking(gfp_mask)))
690 			return (void *)__get_free_page(gfp_mask);
691 
692 		return napi_alloc_frag(fp->rx_frag_size);
693 	}
694 
695 	return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
696 }
697 
698 #ifdef CONFIG_INET
bnx2x_gro_ip_csum(struct bnx2x * bp,struct sk_buff * skb)699 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
700 {
701 	const struct iphdr *iph = ip_hdr(skb);
702 	struct tcphdr *th;
703 
704 	skb_set_transport_header(skb, sizeof(struct iphdr));
705 	th = tcp_hdr(skb);
706 
707 	th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
708 				  iph->saddr, iph->daddr, 0);
709 }
710 
bnx2x_gro_ipv6_csum(struct bnx2x * bp,struct sk_buff * skb)711 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
712 {
713 	struct ipv6hdr *iph = ipv6_hdr(skb);
714 	struct tcphdr *th;
715 
716 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
717 	th = tcp_hdr(skb);
718 
719 	th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
720 				  &iph->saddr, &iph->daddr, 0);
721 }
722 
bnx2x_gro_csum(struct bnx2x * bp,struct sk_buff * skb,void (* gro_func)(struct bnx2x *,struct sk_buff *))723 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
724 			    void (*gro_func)(struct bnx2x*, struct sk_buff*))
725 {
726 	skb_reset_network_header(skb);
727 	gro_func(bp, skb);
728 	tcp_gro_complete(skb);
729 }
730 #endif
731 
bnx2x_gro_receive(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct sk_buff * skb)732 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
733 			       struct sk_buff *skb)
734 {
735 #ifdef CONFIG_INET
736 	if (skb_shinfo(skb)->gso_size) {
737 		switch (be16_to_cpu(skb->protocol)) {
738 		case ETH_P_IP:
739 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
740 			break;
741 		case ETH_P_IPV6:
742 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
743 			break;
744 		default:
745 			netdev_WARN_ONCE(bp->dev,
746 					 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
747 					 be16_to_cpu(skb->protocol));
748 		}
749 	}
750 #endif
751 	skb_record_rx_queue(skb, fp->rx_queue);
752 	napi_gro_receive(&fp->napi, skb);
753 }
754 
bnx2x_tpa_stop(struct bnx2x * bp,struct bnx2x_fastpath * fp,struct bnx2x_agg_info * tpa_info,u16 pages,struct eth_end_agg_rx_cqe * cqe,u16 cqe_idx)755 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
756 			   struct bnx2x_agg_info *tpa_info,
757 			   u16 pages,
758 			   struct eth_end_agg_rx_cqe *cqe,
759 			   u16 cqe_idx)
760 {
761 	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
762 	u8 pad = tpa_info->placement_offset;
763 	u16 len = tpa_info->len_on_bd;
764 	struct sk_buff *skb = NULL;
765 	u8 *new_data, *data = rx_buf->data;
766 	u8 old_tpa_state = tpa_info->tpa_state;
767 
768 	tpa_info->tpa_state = BNX2X_TPA_STOP;
769 
770 	/* If we there was an error during the handling of the TPA_START -
771 	 * drop this aggregation.
772 	 */
773 	if (old_tpa_state == BNX2X_TPA_ERROR)
774 		goto drop;
775 
776 	/* Try to allocate the new data */
777 	new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
778 	/* Unmap skb in the pool anyway, as we are going to change
779 	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
780 	   fails. */
781 	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
782 			 fp->rx_buf_size, DMA_FROM_DEVICE);
783 	if (likely(new_data))
784 		skb = build_skb(data, fp->rx_frag_size);
785 
786 	if (likely(skb)) {
787 #ifdef BNX2X_STOP_ON_ERROR
788 		if (pad + len > fp->rx_buf_size) {
789 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
790 				  pad, len, fp->rx_buf_size);
791 			bnx2x_panic();
792 			return;
793 		}
794 #endif
795 
796 		skb_reserve(skb, pad + NET_SKB_PAD);
797 		skb_put(skb, len);
798 		skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
799 
800 		skb->protocol = eth_type_trans(skb, bp->dev);
801 		skb->ip_summed = CHECKSUM_UNNECESSARY;
802 
803 		if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
804 					 skb, cqe, cqe_idx)) {
805 			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
806 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
807 			bnx2x_gro_receive(bp, fp, skb);
808 		} else {
809 			DP(NETIF_MSG_RX_STATUS,
810 			   "Failed to allocate new pages - dropping packet!\n");
811 			dev_kfree_skb_any(skb);
812 		}
813 
814 		/* put new data in bin */
815 		rx_buf->data = new_data;
816 
817 		return;
818 	}
819 	if (new_data)
820 		bnx2x_frag_free(fp, new_data);
821 drop:
822 	/* drop the packet and keep the buffer in the bin */
823 	DP(NETIF_MSG_RX_STATUS,
824 	   "Failed to allocate or map a new skb - dropping packet!\n");
825 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
826 }
827 
bnx2x_alloc_rx_data(struct bnx2x * bp,struct bnx2x_fastpath * fp,u16 index,gfp_t gfp_mask)828 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
829 			       u16 index, gfp_t gfp_mask)
830 {
831 	u8 *data;
832 	struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
833 	struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
834 	dma_addr_t mapping;
835 
836 	data = bnx2x_frag_alloc(fp, gfp_mask);
837 	if (unlikely(data == NULL))
838 		return -ENOMEM;
839 
840 	mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
841 				 fp->rx_buf_size,
842 				 DMA_FROM_DEVICE);
843 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
844 		bnx2x_frag_free(fp, data);
845 		BNX2X_ERR("Can't map rx data\n");
846 		return -ENOMEM;
847 	}
848 
849 	rx_buf->data = data;
850 	dma_unmap_addr_set(rx_buf, mapping, mapping);
851 
852 	rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
853 	rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
854 
855 	return 0;
856 }
857 
858 static
bnx2x_csum_validate(struct sk_buff * skb,union eth_rx_cqe * cqe,struct bnx2x_fastpath * fp,struct bnx2x_eth_q_stats * qstats)859 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
860 				 struct bnx2x_fastpath *fp,
861 				 struct bnx2x_eth_q_stats *qstats)
862 {
863 	/* Do nothing if no L4 csum validation was done.
864 	 * We do not check whether IP csum was validated. For IPv4 we assume
865 	 * that if the card got as far as validating the L4 csum, it also
866 	 * validated the IP csum. IPv6 has no IP csum.
867 	 */
868 	if (cqe->fast_path_cqe.status_flags &
869 	    ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
870 		return;
871 
872 	/* If L4 validation was done, check if an error was found. */
873 
874 	if (cqe->fast_path_cqe.type_error_flags &
875 	    (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
876 	     ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
877 		qstats->hw_csum_err++;
878 	else
879 		skb->ip_summed = CHECKSUM_UNNECESSARY;
880 }
881 
bnx2x_rx_int(struct bnx2x_fastpath * fp,int budget)882 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
883 {
884 	struct bnx2x *bp = fp->bp;
885 	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
886 	u16 sw_comp_cons, sw_comp_prod;
887 	int rx_pkt = 0;
888 	union eth_rx_cqe *cqe;
889 	struct eth_fast_path_rx_cqe *cqe_fp;
890 
891 #ifdef BNX2X_STOP_ON_ERROR
892 	if (unlikely(bp->panic))
893 		return 0;
894 #endif
895 	if (budget <= 0)
896 		return rx_pkt;
897 
898 	bd_cons = fp->rx_bd_cons;
899 	bd_prod = fp->rx_bd_prod;
900 	bd_prod_fw = bd_prod;
901 	sw_comp_cons = fp->rx_comp_cons;
902 	sw_comp_prod = fp->rx_comp_prod;
903 
904 	comp_ring_cons = RCQ_BD(sw_comp_cons);
905 	cqe = &fp->rx_comp_ring[comp_ring_cons];
906 	cqe_fp = &cqe->fast_path_cqe;
907 
908 	DP(NETIF_MSG_RX_STATUS,
909 	   "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
910 
911 	while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
912 		struct sw_rx_bd *rx_buf = NULL;
913 		struct sk_buff *skb;
914 		u8 cqe_fp_flags;
915 		enum eth_rx_cqe_type cqe_fp_type;
916 		u16 len, pad, queue;
917 		u8 *data;
918 		u32 rxhash;
919 		enum pkt_hash_types rxhash_type;
920 
921 #ifdef BNX2X_STOP_ON_ERROR
922 		if (unlikely(bp->panic))
923 			return 0;
924 #endif
925 
926 		bd_prod = RX_BD(bd_prod);
927 		bd_cons = RX_BD(bd_cons);
928 
929 		/* A rmb() is required to ensure that the CQE is not read
930 		 * before it is written by the adapter DMA.  PCI ordering
931 		 * rules will make sure the other fields are written before
932 		 * the marker at the end of struct eth_fast_path_rx_cqe
933 		 * but without rmb() a weakly ordered processor can process
934 		 * stale data.  Without the barrier TPA state-machine might
935 		 * enter inconsistent state and kernel stack might be
936 		 * provided with incorrect packet description - these lead
937 		 * to various kernel crashed.
938 		 */
939 		rmb();
940 
941 		cqe_fp_flags = cqe_fp->type_error_flags;
942 		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
943 
944 		DP(NETIF_MSG_RX_STATUS,
945 		   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
946 		   CQE_TYPE(cqe_fp_flags),
947 		   cqe_fp_flags, cqe_fp->status_flags,
948 		   le32_to_cpu(cqe_fp->rss_hash_result),
949 		   le16_to_cpu(cqe_fp->vlan_tag),
950 		   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
951 
952 		/* is this a slowpath msg? */
953 		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
954 			bnx2x_sp_event(fp, cqe);
955 			goto next_cqe;
956 		}
957 
958 		rx_buf = &fp->rx_buf_ring[bd_cons];
959 		data = rx_buf->data;
960 
961 		if (!CQE_TYPE_FAST(cqe_fp_type)) {
962 			struct bnx2x_agg_info *tpa_info;
963 			u16 frag_size, pages;
964 #ifdef BNX2X_STOP_ON_ERROR
965 			/* sanity check */
966 			if (fp->mode == TPA_MODE_DISABLED &&
967 			    (CQE_TYPE_START(cqe_fp_type) ||
968 			     CQE_TYPE_STOP(cqe_fp_type)))
969 				BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
970 					  CQE_TYPE(cqe_fp_type));
971 #endif
972 
973 			if (CQE_TYPE_START(cqe_fp_type)) {
974 				u16 queue = cqe_fp->queue_index;
975 				DP(NETIF_MSG_RX_STATUS,
976 				   "calling tpa_start on queue %d\n",
977 				   queue);
978 
979 				bnx2x_tpa_start(fp, queue,
980 						bd_cons, bd_prod,
981 						cqe_fp);
982 
983 				goto next_rx;
984 			}
985 			queue = cqe->end_agg_cqe.queue_index;
986 			tpa_info = &fp->tpa_info[queue];
987 			DP(NETIF_MSG_RX_STATUS,
988 			   "calling tpa_stop on queue %d\n",
989 			   queue);
990 
991 			frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
992 				    tpa_info->len_on_bd;
993 
994 			if (fp->mode == TPA_MODE_GRO)
995 				pages = (frag_size + tpa_info->full_page - 1) /
996 					 tpa_info->full_page;
997 			else
998 				pages = SGE_PAGE_ALIGN(frag_size) >>
999 					SGE_PAGE_SHIFT;
1000 
1001 			bnx2x_tpa_stop(bp, fp, tpa_info, pages,
1002 				       &cqe->end_agg_cqe, comp_ring_cons);
1003 #ifdef BNX2X_STOP_ON_ERROR
1004 			if (bp->panic)
1005 				return 0;
1006 #endif
1007 
1008 			bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1009 			goto next_cqe;
1010 		}
1011 		/* non TPA */
1012 		len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1013 		pad = cqe_fp->placement_offset;
1014 		dma_sync_single_for_cpu(&bp->pdev->dev,
1015 					dma_unmap_addr(rx_buf, mapping),
1016 					pad + RX_COPY_THRESH,
1017 					DMA_FROM_DEVICE);
1018 		pad += NET_SKB_PAD;
1019 		prefetch(data + pad); /* speedup eth_type_trans() */
1020 		/* is this an error packet? */
1021 		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1022 			DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1023 			   "ERROR  flags %x  rx packet %u\n",
1024 			   cqe_fp_flags, sw_comp_cons);
1025 			bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1026 			goto reuse_rx;
1027 		}
1028 
1029 		/* Since we don't have a jumbo ring
1030 		 * copy small packets if mtu > 1500
1031 		 */
1032 		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1033 		    (len <= RX_COPY_THRESH)) {
1034 			skb = napi_alloc_skb(&fp->napi, len);
1035 			if (skb == NULL) {
1036 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1037 				   "ERROR  packet dropped because of alloc failure\n");
1038 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1039 				goto reuse_rx;
1040 			}
1041 			memcpy(skb->data, data + pad, len);
1042 			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1043 		} else {
1044 			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1045 						       GFP_ATOMIC) == 0)) {
1046 				dma_unmap_single(&bp->pdev->dev,
1047 						 dma_unmap_addr(rx_buf, mapping),
1048 						 fp->rx_buf_size,
1049 						 DMA_FROM_DEVICE);
1050 				skb = build_skb(data, fp->rx_frag_size);
1051 				if (unlikely(!skb)) {
1052 					bnx2x_frag_free(fp, data);
1053 					bnx2x_fp_qstats(bp, fp)->
1054 							rx_skb_alloc_failed++;
1055 					goto next_rx;
1056 				}
1057 				skb_reserve(skb, pad);
1058 			} else {
1059 				DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1060 				   "ERROR  packet dropped because of alloc failure\n");
1061 				bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1062 reuse_rx:
1063 				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1064 				goto next_rx;
1065 			}
1066 		}
1067 
1068 		skb_put(skb, len);
1069 		skb->protocol = eth_type_trans(skb, bp->dev);
1070 
1071 		/* Set Toeplitz hash for a none-LRO skb */
1072 		rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1073 		skb_set_hash(skb, rxhash, rxhash_type);
1074 
1075 		skb_checksum_none_assert(skb);
1076 
1077 		if (bp->dev->features & NETIF_F_RXCSUM)
1078 			bnx2x_csum_validate(skb, cqe, fp,
1079 					    bnx2x_fp_qstats(bp, fp));
1080 
1081 		skb_record_rx_queue(skb, fp->rx_queue);
1082 
1083 		/* Check if this packet was timestamped */
1084 		if (unlikely(cqe->fast_path_cqe.type_error_flags &
1085 			     (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1086 			bnx2x_set_rx_ts(bp, skb);
1087 
1088 		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1089 		    PARSING_FLAGS_VLAN)
1090 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1091 					       le16_to_cpu(cqe_fp->vlan_tag));
1092 
1093 		napi_gro_receive(&fp->napi, skb);
1094 next_rx:
1095 		rx_buf->data = NULL;
1096 
1097 		bd_cons = NEXT_RX_IDX(bd_cons);
1098 		bd_prod = NEXT_RX_IDX(bd_prod);
1099 		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1100 		rx_pkt++;
1101 next_cqe:
1102 		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1103 		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1104 
1105 		/* mark CQE as free */
1106 		BNX2X_SEED_CQE(cqe_fp);
1107 
1108 		if (rx_pkt == budget)
1109 			break;
1110 
1111 		comp_ring_cons = RCQ_BD(sw_comp_cons);
1112 		cqe = &fp->rx_comp_ring[comp_ring_cons];
1113 		cqe_fp = &cqe->fast_path_cqe;
1114 	} /* while */
1115 
1116 	fp->rx_bd_cons = bd_cons;
1117 	fp->rx_bd_prod = bd_prod_fw;
1118 	fp->rx_comp_cons = sw_comp_cons;
1119 	fp->rx_comp_prod = sw_comp_prod;
1120 
1121 	/* Update producers */
1122 	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1123 			     fp->rx_sge_prod);
1124 
1125 	return rx_pkt;
1126 }
1127 
bnx2x_msix_fp_int(int irq,void * fp_cookie)1128 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1129 {
1130 	struct bnx2x_fastpath *fp = fp_cookie;
1131 	struct bnx2x *bp = fp->bp;
1132 	u8 cos;
1133 
1134 	DP(NETIF_MSG_INTR,
1135 	   "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1136 	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
1137 
1138 	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1139 
1140 #ifdef BNX2X_STOP_ON_ERROR
1141 	if (unlikely(bp->panic))
1142 		return IRQ_HANDLED;
1143 #endif
1144 
1145 	/* Handle Rx and Tx according to MSI-X vector */
1146 	for_each_cos_in_tx_queue(fp, cos)
1147 		prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1148 
1149 	prefetch(&fp->sb_running_index[SM_RX_ID]);
1150 	napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1151 
1152 	return IRQ_HANDLED;
1153 }
1154 
1155 /* HW Lock for shared dual port PHYs */
bnx2x_acquire_phy_lock(struct bnx2x * bp)1156 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1157 {
1158 	mutex_lock(&bp->port.phy_mutex);
1159 
1160 	bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1161 }
1162 
bnx2x_release_phy_lock(struct bnx2x * bp)1163 void bnx2x_release_phy_lock(struct bnx2x *bp)
1164 {
1165 	bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1166 
1167 	mutex_unlock(&bp->port.phy_mutex);
1168 }
1169 
1170 /* calculates MF speed according to current linespeed and MF configuration */
bnx2x_get_mf_speed(struct bnx2x * bp)1171 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1172 {
1173 	u16 line_speed = bp->link_vars.line_speed;
1174 	if (IS_MF(bp)) {
1175 		u16 maxCfg = bnx2x_extract_max_cfg(bp,
1176 						   bp->mf_config[BP_VN(bp)]);
1177 
1178 		/* Calculate the current MAX line speed limit for the MF
1179 		 * devices
1180 		 */
1181 		if (IS_MF_PERCENT_BW(bp))
1182 			line_speed = (line_speed * maxCfg) / 100;
1183 		else { /* SD mode */
1184 			u16 vn_max_rate = maxCfg * 100;
1185 
1186 			if (vn_max_rate < line_speed)
1187 				line_speed = vn_max_rate;
1188 		}
1189 	}
1190 
1191 	return line_speed;
1192 }
1193 
1194 /**
1195  * bnx2x_fill_report_data - fill link report data to report
1196  *
1197  * @bp:		driver handle
1198  * @data:	link state to update
1199  *
1200  * It uses a none-atomic bit operations because is called under the mutex.
1201  */
bnx2x_fill_report_data(struct bnx2x * bp,struct bnx2x_link_report_data * data)1202 static void bnx2x_fill_report_data(struct bnx2x *bp,
1203 				   struct bnx2x_link_report_data *data)
1204 {
1205 	memset(data, 0, sizeof(*data));
1206 
1207 	if (IS_PF(bp)) {
1208 		/* Fill the report data: effective line speed */
1209 		data->line_speed = bnx2x_get_mf_speed(bp);
1210 
1211 		/* Link is down */
1212 		if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1213 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1214 				  &data->link_report_flags);
1215 
1216 		if (!BNX2X_NUM_ETH_QUEUES(bp))
1217 			__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1218 				  &data->link_report_flags);
1219 
1220 		/* Full DUPLEX */
1221 		if (bp->link_vars.duplex == DUPLEX_FULL)
1222 			__set_bit(BNX2X_LINK_REPORT_FD,
1223 				  &data->link_report_flags);
1224 
1225 		/* Rx Flow Control is ON */
1226 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1227 			__set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1228 				  &data->link_report_flags);
1229 
1230 		/* Tx Flow Control is ON */
1231 		if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1232 			__set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1233 				  &data->link_report_flags);
1234 	} else { /* VF */
1235 		*data = bp->vf_link_vars;
1236 	}
1237 }
1238 
1239 /**
1240  * bnx2x_link_report - report link status to OS.
1241  *
1242  * @bp:		driver handle
1243  *
1244  * Calls the __bnx2x_link_report() under the same locking scheme
1245  * as a link/PHY state managing code to ensure a consistent link
1246  * reporting.
1247  */
1248 
bnx2x_link_report(struct bnx2x * bp)1249 void bnx2x_link_report(struct bnx2x *bp)
1250 {
1251 	bnx2x_acquire_phy_lock(bp);
1252 	__bnx2x_link_report(bp);
1253 	bnx2x_release_phy_lock(bp);
1254 }
1255 
1256 /**
1257  * __bnx2x_link_report - report link status to OS.
1258  *
1259  * @bp:		driver handle
1260  *
1261  * None atomic implementation.
1262  * Should be called under the phy_lock.
1263  */
__bnx2x_link_report(struct bnx2x * bp)1264 void __bnx2x_link_report(struct bnx2x *bp)
1265 {
1266 	struct bnx2x_link_report_data cur_data;
1267 
1268 	if (bp->force_link_down) {
1269 		bp->link_vars.link_up = 0;
1270 		return;
1271 	}
1272 
1273 	/* reread mf_cfg */
1274 	if (IS_PF(bp) && !CHIP_IS_E1(bp))
1275 		bnx2x_read_mf_cfg(bp);
1276 
1277 	/* Read the current link report info */
1278 	bnx2x_fill_report_data(bp, &cur_data);
1279 
1280 	/* Don't report link down or exactly the same link status twice */
1281 	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1282 	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1283 		      &bp->last_reported_link.link_report_flags) &&
1284 	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1285 		      &cur_data.link_report_flags)))
1286 		return;
1287 
1288 	bp->link_cnt++;
1289 
1290 	/* We are going to report a new link parameters now -
1291 	 * remember the current data for the next time.
1292 	 */
1293 	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1294 
1295 	/* propagate status to VFs */
1296 	if (IS_PF(bp))
1297 		bnx2x_iov_link_update(bp);
1298 
1299 	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1300 		     &cur_data.link_report_flags)) {
1301 		netif_carrier_off(bp->dev);
1302 		netdev_err(bp->dev, "NIC Link is Down\n");
1303 		return;
1304 	} else {
1305 		const char *duplex;
1306 		const char *flow;
1307 
1308 		netif_carrier_on(bp->dev);
1309 
1310 		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1311 				       &cur_data.link_report_flags))
1312 			duplex = "full";
1313 		else
1314 			duplex = "half";
1315 
1316 		/* Handle the FC at the end so that only these flags would be
1317 		 * possibly set. This way we may easily check if there is no FC
1318 		 * enabled.
1319 		 */
1320 		if (cur_data.link_report_flags) {
1321 			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1322 				     &cur_data.link_report_flags)) {
1323 				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1324 				     &cur_data.link_report_flags))
1325 					flow = "ON - receive & transmit";
1326 				else
1327 					flow = "ON - receive";
1328 			} else {
1329 				flow = "ON - transmit";
1330 			}
1331 		} else {
1332 			flow = "none";
1333 		}
1334 		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1335 			    cur_data.line_speed, duplex, flow);
1336 	}
1337 }
1338 
bnx2x_set_next_page_sgl(struct bnx2x_fastpath * fp)1339 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1340 {
1341 	int i;
1342 
1343 	for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1344 		struct eth_rx_sge *sge;
1345 
1346 		sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1347 		sge->addr_hi =
1348 			cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1349 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1350 
1351 		sge->addr_lo =
1352 			cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1353 			BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1354 	}
1355 }
1356 
bnx2x_free_tpa_pool(struct bnx2x * bp,struct bnx2x_fastpath * fp,int last)1357 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1358 				struct bnx2x_fastpath *fp, int last)
1359 {
1360 	int i;
1361 
1362 	for (i = 0; i < last; i++) {
1363 		struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1364 		struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1365 		u8 *data = first_buf->data;
1366 
1367 		if (data == NULL) {
1368 			DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1369 			continue;
1370 		}
1371 		if (tpa_info->tpa_state == BNX2X_TPA_START)
1372 			dma_unmap_single(&bp->pdev->dev,
1373 					 dma_unmap_addr(first_buf, mapping),
1374 					 fp->rx_buf_size, DMA_FROM_DEVICE);
1375 		bnx2x_frag_free(fp, data);
1376 		first_buf->data = NULL;
1377 	}
1378 }
1379 
bnx2x_init_rx_rings_cnic(struct bnx2x * bp)1380 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1381 {
1382 	int j;
1383 
1384 	for_each_rx_queue_cnic(bp, j) {
1385 		struct bnx2x_fastpath *fp = &bp->fp[j];
1386 
1387 		fp->rx_bd_cons = 0;
1388 
1389 		/* Activate BD ring */
1390 		/* Warning!
1391 		 * this will generate an interrupt (to the TSTORM)
1392 		 * must only be done after chip is initialized
1393 		 */
1394 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1395 				     fp->rx_sge_prod);
1396 	}
1397 }
1398 
bnx2x_init_rx_rings(struct bnx2x * bp)1399 void bnx2x_init_rx_rings(struct bnx2x *bp)
1400 {
1401 	int func = BP_FUNC(bp);
1402 	u16 ring_prod;
1403 	int i, j;
1404 
1405 	/* Allocate TPA resources */
1406 	for_each_eth_queue(bp, j) {
1407 		struct bnx2x_fastpath *fp = &bp->fp[j];
1408 
1409 		DP(NETIF_MSG_IFUP,
1410 		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1411 
1412 		if (fp->mode != TPA_MODE_DISABLED) {
1413 			/* Fill the per-aggregation pool */
1414 			for (i = 0; i < MAX_AGG_QS(bp); i++) {
1415 				struct bnx2x_agg_info *tpa_info =
1416 					&fp->tpa_info[i];
1417 				struct sw_rx_bd *first_buf =
1418 					&tpa_info->first_buf;
1419 
1420 				first_buf->data =
1421 					bnx2x_frag_alloc(fp, GFP_KERNEL);
1422 				if (!first_buf->data) {
1423 					BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1424 						  j);
1425 					bnx2x_free_tpa_pool(bp, fp, i);
1426 					fp->mode = TPA_MODE_DISABLED;
1427 					break;
1428 				}
1429 				dma_unmap_addr_set(first_buf, mapping, 0);
1430 				tpa_info->tpa_state = BNX2X_TPA_STOP;
1431 			}
1432 
1433 			/* "next page" elements initialization */
1434 			bnx2x_set_next_page_sgl(fp);
1435 
1436 			/* set SGEs bit mask */
1437 			bnx2x_init_sge_ring_bit_mask(fp);
1438 
1439 			/* Allocate SGEs and initialize the ring elements */
1440 			for (i = 0, ring_prod = 0;
1441 			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1442 
1443 				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1444 						       GFP_KERNEL) < 0) {
1445 					BNX2X_ERR("was only able to allocate %d rx sges\n",
1446 						  i);
1447 					BNX2X_ERR("disabling TPA for queue[%d]\n",
1448 						  j);
1449 					/* Cleanup already allocated elements */
1450 					bnx2x_free_rx_sge_range(bp, fp,
1451 								ring_prod);
1452 					bnx2x_free_tpa_pool(bp, fp,
1453 							    MAX_AGG_QS(bp));
1454 					fp->mode = TPA_MODE_DISABLED;
1455 					ring_prod = 0;
1456 					break;
1457 				}
1458 				ring_prod = NEXT_SGE_IDX(ring_prod);
1459 			}
1460 
1461 			fp->rx_sge_prod = ring_prod;
1462 		}
1463 	}
1464 
1465 	for_each_eth_queue(bp, j) {
1466 		struct bnx2x_fastpath *fp = &bp->fp[j];
1467 
1468 		fp->rx_bd_cons = 0;
1469 
1470 		/* Activate BD ring */
1471 		/* Warning!
1472 		 * this will generate an interrupt (to the TSTORM)
1473 		 * must only be done after chip is initialized
1474 		 */
1475 		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1476 				     fp->rx_sge_prod);
1477 
1478 		if (j != 0)
1479 			continue;
1480 
1481 		if (CHIP_IS_E1(bp)) {
1482 			REG_WR(bp, BAR_USTRORM_INTMEM +
1483 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1484 			       U64_LO(fp->rx_comp_mapping));
1485 			REG_WR(bp, BAR_USTRORM_INTMEM +
1486 			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1487 			       U64_HI(fp->rx_comp_mapping));
1488 		}
1489 	}
1490 }
1491 
bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath * fp)1492 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1493 {
1494 	u8 cos;
1495 	struct bnx2x *bp = fp->bp;
1496 
1497 	for_each_cos_in_tx_queue(fp, cos) {
1498 		struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1499 		unsigned pkts_compl = 0, bytes_compl = 0;
1500 
1501 		u16 sw_prod = txdata->tx_pkt_prod;
1502 		u16 sw_cons = txdata->tx_pkt_cons;
1503 
1504 		while (sw_cons != sw_prod) {
1505 			bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1506 					  &pkts_compl, &bytes_compl);
1507 			sw_cons++;
1508 		}
1509 
1510 		netdev_tx_reset_queue(
1511 			netdev_get_tx_queue(bp->dev,
1512 					    txdata->txq_index));
1513 	}
1514 }
1515 
bnx2x_free_tx_skbs_cnic(struct bnx2x * bp)1516 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1517 {
1518 	int i;
1519 
1520 	for_each_tx_queue_cnic(bp, i) {
1521 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1522 	}
1523 }
1524 
bnx2x_free_tx_skbs(struct bnx2x * bp)1525 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1526 {
1527 	int i;
1528 
1529 	for_each_eth_queue(bp, i) {
1530 		bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1531 	}
1532 }
1533 
bnx2x_free_rx_bds(struct bnx2x_fastpath * fp)1534 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1535 {
1536 	struct bnx2x *bp = fp->bp;
1537 	int i;
1538 
1539 	/* ring wasn't allocated */
1540 	if (fp->rx_buf_ring == NULL)
1541 		return;
1542 
1543 	for (i = 0; i < NUM_RX_BD; i++) {
1544 		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1545 		u8 *data = rx_buf->data;
1546 
1547 		if (data == NULL)
1548 			continue;
1549 		dma_unmap_single(&bp->pdev->dev,
1550 				 dma_unmap_addr(rx_buf, mapping),
1551 				 fp->rx_buf_size, DMA_FROM_DEVICE);
1552 
1553 		rx_buf->data = NULL;
1554 		bnx2x_frag_free(fp, data);
1555 	}
1556 }
1557 
bnx2x_free_rx_skbs_cnic(struct bnx2x * bp)1558 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1559 {
1560 	int j;
1561 
1562 	for_each_rx_queue_cnic(bp, j) {
1563 		bnx2x_free_rx_bds(&bp->fp[j]);
1564 	}
1565 }
1566 
bnx2x_free_rx_skbs(struct bnx2x * bp)1567 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1568 {
1569 	int j;
1570 
1571 	for_each_eth_queue(bp, j) {
1572 		struct bnx2x_fastpath *fp = &bp->fp[j];
1573 
1574 		bnx2x_free_rx_bds(fp);
1575 
1576 		if (fp->mode != TPA_MODE_DISABLED)
1577 			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1578 	}
1579 }
1580 
bnx2x_free_skbs_cnic(struct bnx2x * bp)1581 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1582 {
1583 	bnx2x_free_tx_skbs_cnic(bp);
1584 	bnx2x_free_rx_skbs_cnic(bp);
1585 }
1586 
bnx2x_free_skbs(struct bnx2x * bp)1587 void bnx2x_free_skbs(struct bnx2x *bp)
1588 {
1589 	bnx2x_free_tx_skbs(bp);
1590 	bnx2x_free_rx_skbs(bp);
1591 }
1592 
bnx2x_update_max_mf_config(struct bnx2x * bp,u32 value)1593 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1594 {
1595 	/* load old values */
1596 	u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1597 
1598 	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1599 		/* leave all but MAX value */
1600 		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1601 
1602 		/* set new MAX value */
1603 		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1604 				& FUNC_MF_CFG_MAX_BW_MASK;
1605 
1606 		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1607 	}
1608 }
1609 
1610 /**
1611  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1612  *
1613  * @bp:		driver handle
1614  * @nvecs:	number of vectors to be released
1615  */
bnx2x_free_msix_irqs(struct bnx2x * bp,int nvecs)1616 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1617 {
1618 	int i, offset = 0;
1619 
1620 	if (nvecs == offset)
1621 		return;
1622 
1623 	/* VFs don't have a default SB */
1624 	if (IS_PF(bp)) {
1625 		free_irq(bp->msix_table[offset].vector, bp->dev);
1626 		DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1627 		   bp->msix_table[offset].vector);
1628 		offset++;
1629 	}
1630 
1631 	if (CNIC_SUPPORT(bp)) {
1632 		if (nvecs == offset)
1633 			return;
1634 		offset++;
1635 	}
1636 
1637 	for_each_eth_queue(bp, i) {
1638 		if (nvecs == offset)
1639 			return;
1640 		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1641 		   i, bp->msix_table[offset].vector);
1642 
1643 		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1644 	}
1645 }
1646 
bnx2x_free_irq(struct bnx2x * bp)1647 void bnx2x_free_irq(struct bnx2x *bp)
1648 {
1649 	if (bp->flags & USING_MSIX_FLAG &&
1650 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1651 		int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1652 
1653 		/* vfs don't have a default status block */
1654 		if (IS_PF(bp))
1655 			nvecs++;
1656 
1657 		bnx2x_free_msix_irqs(bp, nvecs);
1658 	} else {
1659 		free_irq(bp->dev->irq, bp->dev);
1660 	}
1661 }
1662 
bnx2x_enable_msix(struct bnx2x * bp)1663 int bnx2x_enable_msix(struct bnx2x *bp)
1664 {
1665 	int msix_vec = 0, i, rc;
1666 
1667 	/* VFs don't have a default status block */
1668 	if (IS_PF(bp)) {
1669 		bp->msix_table[msix_vec].entry = msix_vec;
1670 		BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1671 			       bp->msix_table[0].entry);
1672 		msix_vec++;
1673 	}
1674 
1675 	/* Cnic requires an msix vector for itself */
1676 	if (CNIC_SUPPORT(bp)) {
1677 		bp->msix_table[msix_vec].entry = msix_vec;
1678 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1679 			       msix_vec, bp->msix_table[msix_vec].entry);
1680 		msix_vec++;
1681 	}
1682 
1683 	/* We need separate vectors for ETH queues only (not FCoE) */
1684 	for_each_eth_queue(bp, i) {
1685 		bp->msix_table[msix_vec].entry = msix_vec;
1686 		BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1687 			       msix_vec, msix_vec, i);
1688 		msix_vec++;
1689 	}
1690 
1691 	DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1692 	   msix_vec);
1693 
1694 	rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1695 				   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1696 	/*
1697 	 * reconfigure number of tx/rx queues according to available
1698 	 * MSI-X vectors
1699 	 */
1700 	if (rc == -ENOSPC) {
1701 		/* Get by with single vector */
1702 		rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1703 		if (rc < 0) {
1704 			BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1705 				       rc);
1706 			goto no_msix;
1707 		}
1708 
1709 		BNX2X_DEV_INFO("Using single MSI-X vector\n");
1710 		bp->flags |= USING_SINGLE_MSIX_FLAG;
1711 
1712 		BNX2X_DEV_INFO("set number of queues to 1\n");
1713 		bp->num_ethernet_queues = 1;
1714 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1715 	} else if (rc < 0) {
1716 		BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1717 		goto no_msix;
1718 	} else if (rc < msix_vec) {
1719 		/* how less vectors we will have? */
1720 		int diff = msix_vec - rc;
1721 
1722 		BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1723 
1724 		/*
1725 		 * decrease number of queues by number of unallocated entries
1726 		 */
1727 		bp->num_ethernet_queues -= diff;
1728 		bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1729 
1730 		BNX2X_DEV_INFO("New queue configuration set: %d\n",
1731 			       bp->num_queues);
1732 	}
1733 
1734 	bp->flags |= USING_MSIX_FLAG;
1735 
1736 	return 0;
1737 
1738 no_msix:
1739 	/* fall to INTx if not enough memory */
1740 	if (rc == -ENOMEM)
1741 		bp->flags |= DISABLE_MSI_FLAG;
1742 
1743 	return rc;
1744 }
1745 
bnx2x_req_msix_irqs(struct bnx2x * bp)1746 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1747 {
1748 	int i, rc, offset = 0;
1749 
1750 	/* no default status block for vf */
1751 	if (IS_PF(bp)) {
1752 		rc = request_irq(bp->msix_table[offset++].vector,
1753 				 bnx2x_msix_sp_int, 0,
1754 				 bp->dev->name, bp->dev);
1755 		if (rc) {
1756 			BNX2X_ERR("request sp irq failed\n");
1757 			return -EBUSY;
1758 		}
1759 	}
1760 
1761 	if (CNIC_SUPPORT(bp))
1762 		offset++;
1763 
1764 	for_each_eth_queue(bp, i) {
1765 		struct bnx2x_fastpath *fp = &bp->fp[i];
1766 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1767 			 bp->dev->name, i);
1768 
1769 		rc = request_irq(bp->msix_table[offset].vector,
1770 				 bnx2x_msix_fp_int, 0, fp->name, fp);
1771 		if (rc) {
1772 			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1773 			      bp->msix_table[offset].vector, rc);
1774 			bnx2x_free_msix_irqs(bp, offset);
1775 			return -EBUSY;
1776 		}
1777 
1778 		offset++;
1779 	}
1780 
1781 	i = BNX2X_NUM_ETH_QUEUES(bp);
1782 	if (IS_PF(bp)) {
1783 		offset = 1 + CNIC_SUPPORT(bp);
1784 		netdev_info(bp->dev,
1785 			    "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1786 			    bp->msix_table[0].vector,
1787 			    0, bp->msix_table[offset].vector,
1788 			    i - 1, bp->msix_table[offset + i - 1].vector);
1789 	} else {
1790 		offset = CNIC_SUPPORT(bp);
1791 		netdev_info(bp->dev,
1792 			    "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1793 			    0, bp->msix_table[offset].vector,
1794 			    i - 1, bp->msix_table[offset + i - 1].vector);
1795 	}
1796 	return 0;
1797 }
1798 
bnx2x_enable_msi(struct bnx2x * bp)1799 int bnx2x_enable_msi(struct bnx2x *bp)
1800 {
1801 	int rc;
1802 
1803 	rc = pci_enable_msi(bp->pdev);
1804 	if (rc) {
1805 		BNX2X_DEV_INFO("MSI is not attainable\n");
1806 		return -1;
1807 	}
1808 	bp->flags |= USING_MSI_FLAG;
1809 
1810 	return 0;
1811 }
1812 
bnx2x_req_irq(struct bnx2x * bp)1813 static int bnx2x_req_irq(struct bnx2x *bp)
1814 {
1815 	unsigned long flags;
1816 	unsigned int irq;
1817 
1818 	if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1819 		flags = 0;
1820 	else
1821 		flags = IRQF_SHARED;
1822 
1823 	if (bp->flags & USING_MSIX_FLAG)
1824 		irq = bp->msix_table[0].vector;
1825 	else
1826 		irq = bp->pdev->irq;
1827 
1828 	return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1829 }
1830 
bnx2x_setup_irqs(struct bnx2x * bp)1831 static int bnx2x_setup_irqs(struct bnx2x *bp)
1832 {
1833 	int rc = 0;
1834 	if (bp->flags & USING_MSIX_FLAG &&
1835 	    !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1836 		rc = bnx2x_req_msix_irqs(bp);
1837 		if (rc)
1838 			return rc;
1839 	} else {
1840 		rc = bnx2x_req_irq(bp);
1841 		if (rc) {
1842 			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1843 			return rc;
1844 		}
1845 		if (bp->flags & USING_MSI_FLAG) {
1846 			bp->dev->irq = bp->pdev->irq;
1847 			netdev_info(bp->dev, "using MSI IRQ %d\n",
1848 				    bp->dev->irq);
1849 		}
1850 		if (bp->flags & USING_MSIX_FLAG) {
1851 			bp->dev->irq = bp->msix_table[0].vector;
1852 			netdev_info(bp->dev, "using MSIX IRQ %d\n",
1853 				    bp->dev->irq);
1854 		}
1855 	}
1856 
1857 	return 0;
1858 }
1859 
bnx2x_napi_enable_cnic(struct bnx2x * bp)1860 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1861 {
1862 	int i;
1863 
1864 	for_each_rx_queue_cnic(bp, i) {
1865 		napi_enable(&bnx2x_fp(bp, i, napi));
1866 	}
1867 }
1868 
bnx2x_napi_enable(struct bnx2x * bp)1869 static void bnx2x_napi_enable(struct bnx2x *bp)
1870 {
1871 	int i;
1872 
1873 	for_each_eth_queue(bp, i) {
1874 		napi_enable(&bnx2x_fp(bp, i, napi));
1875 	}
1876 }
1877 
bnx2x_napi_disable_cnic(struct bnx2x * bp)1878 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1879 {
1880 	int i;
1881 
1882 	for_each_rx_queue_cnic(bp, i) {
1883 		napi_disable(&bnx2x_fp(bp, i, napi));
1884 	}
1885 }
1886 
bnx2x_napi_disable(struct bnx2x * bp)1887 static void bnx2x_napi_disable(struct bnx2x *bp)
1888 {
1889 	int i;
1890 
1891 	for_each_eth_queue(bp, i) {
1892 		napi_disable(&bnx2x_fp(bp, i, napi));
1893 	}
1894 }
1895 
bnx2x_netif_start(struct bnx2x * bp)1896 void bnx2x_netif_start(struct bnx2x *bp)
1897 {
1898 	if (netif_running(bp->dev)) {
1899 		bnx2x_napi_enable(bp);
1900 		if (CNIC_LOADED(bp))
1901 			bnx2x_napi_enable_cnic(bp);
1902 		bnx2x_int_enable(bp);
1903 		if (bp->state == BNX2X_STATE_OPEN)
1904 			netif_tx_wake_all_queues(bp->dev);
1905 	}
1906 }
1907 
bnx2x_netif_stop(struct bnx2x * bp,int disable_hw)1908 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1909 {
1910 	bnx2x_int_disable_sync(bp, disable_hw);
1911 	bnx2x_napi_disable(bp);
1912 	if (CNIC_LOADED(bp))
1913 		bnx2x_napi_disable_cnic(bp);
1914 }
1915 
bnx2x_select_queue(struct net_device * dev,struct sk_buff * skb,struct net_device * sb_dev)1916 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1917 		       struct net_device *sb_dev)
1918 {
1919 	struct bnx2x *bp = netdev_priv(dev);
1920 
1921 	if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1922 		struct ethhdr *hdr = (struct ethhdr *)skb->data;
1923 		u16 ether_type = ntohs(hdr->h_proto);
1924 
1925 		/* Skip VLAN tag if present */
1926 		if (ether_type == ETH_P_8021Q) {
1927 			struct vlan_ethhdr *vhdr =
1928 				(struct vlan_ethhdr *)skb->data;
1929 
1930 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1931 		}
1932 
1933 		/* If ethertype is FCoE or FIP - use FCoE ring */
1934 		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1935 			return bnx2x_fcoe_tx(bp, txq_index);
1936 	}
1937 
1938 	/* select a non-FCoE queue */
1939 	return netdev_pick_tx(dev, skb, NULL) %
1940 			(BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos);
1941 }
1942 
bnx2x_set_num_queues(struct bnx2x * bp)1943 void bnx2x_set_num_queues(struct bnx2x *bp)
1944 {
1945 	/* RSS queues */
1946 	bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1947 
1948 	/* override in STORAGE SD modes */
1949 	if (IS_MF_STORAGE_ONLY(bp))
1950 		bp->num_ethernet_queues = 1;
1951 
1952 	/* Add special queues */
1953 	bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1954 	bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1955 
1956 	BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1957 }
1958 
1959 /**
1960  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1961  *
1962  * @bp:		Driver handle
1963  * @include_cnic: handle cnic case
1964  *
1965  * We currently support for at most 16 Tx queues for each CoS thus we will
1966  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1967  * bp->max_cos.
1968  *
1969  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1970  * index after all ETH L2 indices.
1971  *
1972  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1973  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1974  * 16..31,...) with indices that are not coupled with any real Tx queue.
1975  *
1976  * The proper configuration of skb->queue_mapping is handled by
1977  * bnx2x_select_queue() and __skb_tx_hash().
1978  *
1979  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1980  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1981  */
bnx2x_set_real_num_queues(struct bnx2x * bp,int include_cnic)1982 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1983 {
1984 	int rc, tx, rx;
1985 
1986 	tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1987 	rx = BNX2X_NUM_ETH_QUEUES(bp);
1988 
1989 /* account for fcoe queue */
1990 	if (include_cnic && !NO_FCOE(bp)) {
1991 		rx++;
1992 		tx++;
1993 	}
1994 
1995 	rc = netif_set_real_num_tx_queues(bp->dev, tx);
1996 	if (rc) {
1997 		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1998 		return rc;
1999 	}
2000 	rc = netif_set_real_num_rx_queues(bp->dev, rx);
2001 	if (rc) {
2002 		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2003 		return rc;
2004 	}
2005 
2006 	DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2007 			  tx, rx);
2008 
2009 	return rc;
2010 }
2011 
bnx2x_set_rx_buf_size(struct bnx2x * bp)2012 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2013 {
2014 	int i;
2015 
2016 	for_each_queue(bp, i) {
2017 		struct bnx2x_fastpath *fp = &bp->fp[i];
2018 		u32 mtu;
2019 
2020 		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
2021 		if (IS_FCOE_IDX(i))
2022 			/*
2023 			 * Although there are no IP frames expected to arrive to
2024 			 * this ring we still want to add an
2025 			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2026 			 * overrun attack.
2027 			 */
2028 			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2029 		else
2030 			mtu = bp->dev->mtu;
2031 		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2032 				  IP_HEADER_ALIGNMENT_PADDING +
2033 				  ETH_OVERHEAD +
2034 				  mtu +
2035 				  BNX2X_FW_RX_ALIGN_END;
2036 		fp->rx_buf_size = SKB_DATA_ALIGN(fp->rx_buf_size);
2037 		/* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2038 		if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2039 			fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2040 		else
2041 			fp->rx_frag_size = 0;
2042 	}
2043 }
2044 
bnx2x_init_rss(struct bnx2x * bp)2045 static int bnx2x_init_rss(struct bnx2x *bp)
2046 {
2047 	int i;
2048 	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2049 
2050 	/* Prepare the initial contents for the indirection table if RSS is
2051 	 * enabled
2052 	 */
2053 	for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2054 		bp->rss_conf_obj.ind_table[i] =
2055 			bp->fp->cl_id +
2056 			ethtool_rxfh_indir_default(i, num_eth_queues);
2057 
2058 	/*
2059 	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2060 	 * per-port, so if explicit configuration is needed , do it only
2061 	 * for a PMF.
2062 	 *
2063 	 * For 57712 and newer on the other hand it's a per-function
2064 	 * configuration.
2065 	 */
2066 	return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2067 }
2068 
bnx2x_rss(struct bnx2x * bp,struct bnx2x_rss_config_obj * rss_obj,bool config_hash,bool enable)2069 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2070 	      bool config_hash, bool enable)
2071 {
2072 	struct bnx2x_config_rss_params params = {NULL};
2073 
2074 	/* Although RSS is meaningless when there is a single HW queue we
2075 	 * still need it enabled in order to have HW Rx hash generated.
2076 	 *
2077 	 * if (!is_eth_multi(bp))
2078 	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2079 	 */
2080 
2081 	params.rss_obj = rss_obj;
2082 
2083 	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2084 
2085 	if (enable) {
2086 		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2087 
2088 		/* RSS configuration */
2089 		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2090 		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2091 		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2092 		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2093 		if (rss_obj->udp_rss_v4)
2094 			__set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2095 		if (rss_obj->udp_rss_v6)
2096 			__set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2097 
2098 		if (!CHIP_IS_E1x(bp)) {
2099 			/* valid only for TUNN_MODE_VXLAN tunnel mode */
2100 			__set_bit(BNX2X_RSS_IPV4_VXLAN, &params.rss_flags);
2101 			__set_bit(BNX2X_RSS_IPV6_VXLAN, &params.rss_flags);
2102 
2103 			/* valid only for TUNN_MODE_GRE tunnel mode */
2104 			__set_bit(BNX2X_RSS_TUNN_INNER_HDRS, &params.rss_flags);
2105 		}
2106 	} else {
2107 		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2108 	}
2109 
2110 	/* Hash bits */
2111 	params.rss_result_mask = MULTI_MASK;
2112 
2113 	memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2114 
2115 	if (config_hash) {
2116 		/* RSS keys */
2117 		netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2118 		__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2119 	}
2120 
2121 	if (IS_PF(bp))
2122 		return bnx2x_config_rss(bp, &params);
2123 	else
2124 		return bnx2x_vfpf_config_rss(bp, &params);
2125 }
2126 
bnx2x_init_hw(struct bnx2x * bp,u32 load_code)2127 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2128 {
2129 	struct bnx2x_func_state_params func_params = {NULL};
2130 
2131 	/* Prepare parameters for function state transitions */
2132 	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2133 
2134 	func_params.f_obj = &bp->func_obj;
2135 	func_params.cmd = BNX2X_F_CMD_HW_INIT;
2136 
2137 	func_params.params.hw_init.load_phase = load_code;
2138 
2139 	return bnx2x_func_state_change(bp, &func_params);
2140 }
2141 
2142 /*
2143  * Cleans the object that have internal lists without sending
2144  * ramrods. Should be run when interrupts are disabled.
2145  */
bnx2x_squeeze_objects(struct bnx2x * bp)2146 void bnx2x_squeeze_objects(struct bnx2x *bp)
2147 {
2148 	int rc;
2149 	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2150 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
2151 	struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2152 
2153 	/***************** Cleanup MACs' object first *************************/
2154 
2155 	/* Wait for completion of requested */
2156 	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2157 	/* Perform a dry cleanup */
2158 	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2159 
2160 	/* Clean ETH primary MAC */
2161 	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2162 	rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2163 				 &ramrod_flags);
2164 	if (rc != 0)
2165 		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2166 
2167 	/* Cleanup UC list */
2168 	vlan_mac_flags = 0;
2169 	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2170 	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2171 				 &ramrod_flags);
2172 	if (rc != 0)
2173 		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2174 
2175 	/***************** Now clean mcast object *****************************/
2176 	rparam.mcast_obj = &bp->mcast_obj;
2177 	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2178 
2179 	/* Add a DEL command... - Since we're doing a driver cleanup only,
2180 	 * we take a lock surrounding both the initial send and the CONTs,
2181 	 * as we don't want a true completion to disrupt us in the middle.
2182 	 */
2183 	netif_addr_lock_bh(bp->dev);
2184 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2185 	if (rc < 0)
2186 		BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2187 			  rc);
2188 
2189 	/* ...and wait until all pending commands are cleared */
2190 	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2191 	while (rc != 0) {
2192 		if (rc < 0) {
2193 			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2194 				  rc);
2195 			netif_addr_unlock_bh(bp->dev);
2196 			return;
2197 		}
2198 
2199 		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2200 	}
2201 	netif_addr_unlock_bh(bp->dev);
2202 }
2203 
2204 #ifndef BNX2X_STOP_ON_ERROR
2205 #define LOAD_ERROR_EXIT(bp, label) \
2206 	do { \
2207 		(bp)->state = BNX2X_STATE_ERROR; \
2208 		goto label; \
2209 	} while (0)
2210 
2211 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2212 	do { \
2213 		bp->cnic_loaded = false; \
2214 		goto label; \
2215 	} while (0)
2216 #else /*BNX2X_STOP_ON_ERROR*/
2217 #define LOAD_ERROR_EXIT(bp, label) \
2218 	do { \
2219 		(bp)->state = BNX2X_STATE_ERROR; \
2220 		(bp)->panic = 1; \
2221 		return -EBUSY; \
2222 	} while (0)
2223 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2224 	do { \
2225 		bp->cnic_loaded = false; \
2226 		(bp)->panic = 1; \
2227 		return -EBUSY; \
2228 	} while (0)
2229 #endif /*BNX2X_STOP_ON_ERROR*/
2230 
bnx2x_free_fw_stats_mem(struct bnx2x * bp)2231 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2232 {
2233 	BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2234 		       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2235 	return;
2236 }
2237 
bnx2x_alloc_fw_stats_mem(struct bnx2x * bp)2238 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2239 {
2240 	int num_groups, vf_headroom = 0;
2241 	int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2242 
2243 	/* number of queues for statistics is number of eth queues + FCoE */
2244 	u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2245 
2246 	/* Total number of FW statistics requests =
2247 	 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2248 	 * and fcoe l2 queue) stats + num of queues (which includes another 1
2249 	 * for fcoe l2 queue if applicable)
2250 	 */
2251 	bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2252 
2253 	/* vf stats appear in the request list, but their data is allocated by
2254 	 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2255 	 * it is used to determine where to place the vf stats queries in the
2256 	 * request struct
2257 	 */
2258 	if (IS_SRIOV(bp))
2259 		vf_headroom = bnx2x_vf_headroom(bp);
2260 
2261 	/* Request is built from stats_query_header and an array of
2262 	 * stats_query_cmd_group each of which contains
2263 	 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2264 	 * configured in the stats_query_header.
2265 	 */
2266 	num_groups =
2267 		(((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2268 		 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2269 		 1 : 0));
2270 
2271 	DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2272 	   bp->fw_stats_num, vf_headroom, num_groups);
2273 	bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2274 		num_groups * sizeof(struct stats_query_cmd_group);
2275 
2276 	/* Data for statistics requests + stats_counter
2277 	 * stats_counter holds per-STORM counters that are incremented
2278 	 * when STORM has finished with the current request.
2279 	 * memory for FCoE offloaded statistics are counted anyway,
2280 	 * even if they will not be sent.
2281 	 * VF stats are not accounted for here as the data of VF stats is stored
2282 	 * in memory allocated by the VF, not here.
2283 	 */
2284 	bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2285 		sizeof(struct per_pf_stats) +
2286 		sizeof(struct fcoe_statistics_params) +
2287 		sizeof(struct per_queue_stats) * num_queue_stats +
2288 		sizeof(struct stats_counter);
2289 
2290 	bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2291 				       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2292 	if (!bp->fw_stats)
2293 		goto alloc_mem_err;
2294 
2295 	/* Set shortcuts */
2296 	bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2297 	bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2298 	bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2299 		((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2300 	bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2301 		bp->fw_stats_req_sz;
2302 
2303 	DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2304 	   U64_HI(bp->fw_stats_req_mapping),
2305 	   U64_LO(bp->fw_stats_req_mapping));
2306 	DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2307 	   U64_HI(bp->fw_stats_data_mapping),
2308 	   U64_LO(bp->fw_stats_data_mapping));
2309 	return 0;
2310 
2311 alloc_mem_err:
2312 	bnx2x_free_fw_stats_mem(bp);
2313 	BNX2X_ERR("Can't allocate FW stats memory\n");
2314 	return -ENOMEM;
2315 }
2316 
2317 /* send load request to mcp and analyze response */
bnx2x_nic_load_request(struct bnx2x * bp,u32 * load_code)2318 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2319 {
2320 	u32 param;
2321 
2322 	/* init fw_seq */
2323 	bp->fw_seq =
2324 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2325 		 DRV_MSG_SEQ_NUMBER_MASK);
2326 	BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2327 
2328 	/* Get current FW pulse sequence */
2329 	bp->fw_drv_pulse_wr_seq =
2330 		(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2331 		 DRV_PULSE_SEQ_MASK);
2332 	BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2333 
2334 	param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2335 
2336 	if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2337 		param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2338 
2339 	/* load request */
2340 	(*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2341 
2342 	/* if mcp fails to respond we must abort */
2343 	if (!(*load_code)) {
2344 		BNX2X_ERR("MCP response failure, aborting\n");
2345 		return -EBUSY;
2346 	}
2347 
2348 	/* If mcp refused (e.g. other port is in diagnostic mode) we
2349 	 * must abort
2350 	 */
2351 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2352 		BNX2X_ERR("MCP refused load request, aborting\n");
2353 		return -EBUSY;
2354 	}
2355 	return 0;
2356 }
2357 
2358 /* check whether another PF has already loaded FW to chip. In
2359  * virtualized environments a pf from another VM may have already
2360  * initialized the device including loading FW
2361  */
bnx2x_compare_fw_ver(struct bnx2x * bp,u32 load_code,bool print_err)2362 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2363 {
2364 	/* is another pf loaded on this engine? */
2365 	if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2366 	    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2367 		u8 loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng;
2368 		u32 loaded_fw;
2369 
2370 		/* read loaded FW from chip */
2371 		loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2372 
2373 		loaded_fw_major = loaded_fw & 0xff;
2374 		loaded_fw_minor = (loaded_fw >> 8) & 0xff;
2375 		loaded_fw_rev = (loaded_fw >> 16) & 0xff;
2376 		loaded_fw_eng = (loaded_fw >> 24) & 0xff;
2377 
2378 		DP(BNX2X_MSG_SP, "loaded fw 0x%x major 0x%x minor 0x%x rev 0x%x eng 0x%x\n",
2379 		   loaded_fw, loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng);
2380 
2381 		/* abort nic load if version mismatch */
2382 		if (loaded_fw_major != BCM_5710_FW_MAJOR_VERSION ||
2383 		    loaded_fw_minor != BCM_5710_FW_MINOR_VERSION ||
2384 		    loaded_fw_eng != BCM_5710_FW_ENGINEERING_VERSION ||
2385 		    loaded_fw_rev < BCM_5710_FW_REVISION_VERSION_V15) {
2386 			if (print_err)
2387 				BNX2X_ERR("loaded FW incompatible. Aborting\n");
2388 			else
2389 				BNX2X_DEV_INFO("loaded FW incompatible, possibly due to MF UNDI\n");
2390 
2391 			return -EBUSY;
2392 		}
2393 	}
2394 	return 0;
2395 }
2396 
2397 /* returns the "mcp load_code" according to global load_count array */
bnx2x_nic_load_no_mcp(struct bnx2x * bp,int port)2398 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2399 {
2400 	int path = BP_PATH(bp);
2401 
2402 	DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2403 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2404 	   bnx2x_load_count[path][2]);
2405 	bnx2x_load_count[path][0]++;
2406 	bnx2x_load_count[path][1 + port]++;
2407 	DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2408 	   path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2409 	   bnx2x_load_count[path][2]);
2410 	if (bnx2x_load_count[path][0] == 1)
2411 		return FW_MSG_CODE_DRV_LOAD_COMMON;
2412 	else if (bnx2x_load_count[path][1 + port] == 1)
2413 		return FW_MSG_CODE_DRV_LOAD_PORT;
2414 	else
2415 		return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2416 }
2417 
2418 /* mark PMF if applicable */
bnx2x_nic_load_pmf(struct bnx2x * bp,u32 load_code)2419 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2420 {
2421 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2422 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2423 	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2424 		bp->port.pmf = 1;
2425 		/* We need the barrier to ensure the ordering between the
2426 		 * writing to bp->port.pmf here and reading it from the
2427 		 * bnx2x_periodic_task().
2428 		 */
2429 		smp_mb();
2430 	} else {
2431 		bp->port.pmf = 0;
2432 	}
2433 
2434 	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2435 }
2436 
bnx2x_nic_load_afex_dcc(struct bnx2x * bp,int load_code)2437 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2438 {
2439 	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2440 	     (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2441 	    (bp->common.shmem2_base)) {
2442 		if (SHMEM2_HAS(bp, dcc_support))
2443 			SHMEM2_WR(bp, dcc_support,
2444 				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2445 				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2446 		if (SHMEM2_HAS(bp, afex_driver_support))
2447 			SHMEM2_WR(bp, afex_driver_support,
2448 				  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2449 	}
2450 
2451 	/* Set AFEX default VLAN tag to an invalid value */
2452 	bp->afex_def_vlan_tag = -1;
2453 }
2454 
2455 /**
2456  * bnx2x_bz_fp - zero content of the fastpath structure.
2457  *
2458  * @bp:		driver handle
2459  * @index:	fastpath index to be zeroed
2460  *
2461  * Makes sure the contents of the bp->fp[index].napi is kept
2462  * intact.
2463  */
bnx2x_bz_fp(struct bnx2x * bp,int index)2464 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2465 {
2466 	struct bnx2x_fastpath *fp = &bp->fp[index];
2467 	int cos;
2468 	struct napi_struct orig_napi = fp->napi;
2469 	struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2470 
2471 	/* bzero bnx2x_fastpath contents */
2472 	if (fp->tpa_info)
2473 		memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2474 		       sizeof(struct bnx2x_agg_info));
2475 	memset(fp, 0, sizeof(*fp));
2476 
2477 	/* Restore the NAPI object as it has been already initialized */
2478 	fp->napi = orig_napi;
2479 	fp->tpa_info = orig_tpa_info;
2480 	fp->bp = bp;
2481 	fp->index = index;
2482 	if (IS_ETH_FP(fp))
2483 		fp->max_cos = bp->max_cos;
2484 	else
2485 		/* Special queues support only one CoS */
2486 		fp->max_cos = 1;
2487 
2488 	/* Init txdata pointers */
2489 	if (IS_FCOE_FP(fp))
2490 		fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2491 	if (IS_ETH_FP(fp))
2492 		for_each_cos_in_tx_queue(fp, cos)
2493 			fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2494 				BNX2X_NUM_ETH_QUEUES(bp) + index];
2495 
2496 	/* set the tpa flag for each queue. The tpa flag determines the queue
2497 	 * minimal size so it must be set prior to queue memory allocation
2498 	 */
2499 	if (bp->dev->features & NETIF_F_LRO)
2500 		fp->mode = TPA_MODE_LRO;
2501 	else if (bp->dev->features & NETIF_F_GRO_HW)
2502 		fp->mode = TPA_MODE_GRO;
2503 	else
2504 		fp->mode = TPA_MODE_DISABLED;
2505 
2506 	/* We don't want TPA if it's disabled in bp
2507 	 * or if this is an FCoE L2 ring.
2508 	 */
2509 	if (bp->disable_tpa || IS_FCOE_FP(fp))
2510 		fp->mode = TPA_MODE_DISABLED;
2511 }
2512 
bnx2x_set_os_driver_state(struct bnx2x * bp,u32 state)2513 void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state)
2514 {
2515 	u32 cur;
2516 
2517 	if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp))
2518 		return;
2519 
2520 	cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]);
2521 	DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n",
2522 	   cur, state);
2523 
2524 	SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state);
2525 }
2526 
bnx2x_load_cnic(struct bnx2x * bp)2527 int bnx2x_load_cnic(struct bnx2x *bp)
2528 {
2529 	int i, rc, port = BP_PORT(bp);
2530 
2531 	DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2532 
2533 	mutex_init(&bp->cnic_mutex);
2534 
2535 	if (IS_PF(bp)) {
2536 		rc = bnx2x_alloc_mem_cnic(bp);
2537 		if (rc) {
2538 			BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2539 			LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2540 		}
2541 	}
2542 
2543 	rc = bnx2x_alloc_fp_mem_cnic(bp);
2544 	if (rc) {
2545 		BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2546 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2547 	}
2548 
2549 	/* Update the number of queues with the cnic queues */
2550 	rc = bnx2x_set_real_num_queues(bp, 1);
2551 	if (rc) {
2552 		BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2553 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2554 	}
2555 
2556 	/* Add all CNIC NAPI objects */
2557 	bnx2x_add_all_napi_cnic(bp);
2558 	DP(NETIF_MSG_IFUP, "cnic napi added\n");
2559 	bnx2x_napi_enable_cnic(bp);
2560 
2561 	rc = bnx2x_init_hw_func_cnic(bp);
2562 	if (rc)
2563 		LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2564 
2565 	bnx2x_nic_init_cnic(bp);
2566 
2567 	if (IS_PF(bp)) {
2568 		/* Enable Timer scan */
2569 		REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2570 
2571 		/* setup cnic queues */
2572 		for_each_cnic_queue(bp, i) {
2573 			rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2574 			if (rc) {
2575 				BNX2X_ERR("Queue setup failed\n");
2576 				LOAD_ERROR_EXIT(bp, load_error_cnic2);
2577 			}
2578 		}
2579 	}
2580 
2581 	/* Initialize Rx filter. */
2582 	bnx2x_set_rx_mode_inner(bp);
2583 
2584 	/* re-read iscsi info */
2585 	bnx2x_get_iscsi_info(bp);
2586 	bnx2x_setup_cnic_irq_info(bp);
2587 	bnx2x_setup_cnic_info(bp);
2588 	bp->cnic_loaded = true;
2589 	if (bp->state == BNX2X_STATE_OPEN)
2590 		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2591 
2592 	DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2593 
2594 	return 0;
2595 
2596 #ifndef BNX2X_STOP_ON_ERROR
2597 load_error_cnic2:
2598 	/* Disable Timer scan */
2599 	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2600 
2601 load_error_cnic1:
2602 	bnx2x_napi_disable_cnic(bp);
2603 	/* Update the number of queues without the cnic queues */
2604 	if (bnx2x_set_real_num_queues(bp, 0))
2605 		BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2606 load_error_cnic0:
2607 	BNX2X_ERR("CNIC-related load failed\n");
2608 	bnx2x_free_fp_mem_cnic(bp);
2609 	bnx2x_free_mem_cnic(bp);
2610 	return rc;
2611 #endif /* ! BNX2X_STOP_ON_ERROR */
2612 }
2613 
2614 /* must be called with rtnl_lock */
bnx2x_nic_load(struct bnx2x * bp,int load_mode)2615 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2616 {
2617 	int port = BP_PORT(bp);
2618 	int i, rc = 0, load_code = 0;
2619 
2620 	DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2621 	DP(NETIF_MSG_IFUP,
2622 	   "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2623 
2624 #ifdef BNX2X_STOP_ON_ERROR
2625 	if (unlikely(bp->panic)) {
2626 		BNX2X_ERR("Can't load NIC when there is panic\n");
2627 		return -EPERM;
2628 	}
2629 #endif
2630 
2631 	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2632 
2633 	/* zero the structure w/o any lock, before SP handler is initialized */
2634 	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2635 	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2636 		&bp->last_reported_link.link_report_flags);
2637 
2638 	if (IS_PF(bp))
2639 		/* must be called before memory allocation and HW init */
2640 		bnx2x_ilt_set_info(bp);
2641 
2642 	/*
2643 	 * Zero fastpath structures preserving invariants like napi, which are
2644 	 * allocated only once, fp index, max_cos, bp pointer.
2645 	 * Also set fp->mode and txdata_ptr.
2646 	 */
2647 	DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2648 	for_each_queue(bp, i)
2649 		bnx2x_bz_fp(bp, i);
2650 	memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2651 				  bp->num_cnic_queues) *
2652 				  sizeof(struct bnx2x_fp_txdata));
2653 
2654 	bp->fcoe_init = false;
2655 
2656 	/* Set the receive queues buffer size */
2657 	bnx2x_set_rx_buf_size(bp);
2658 
2659 	if (IS_PF(bp)) {
2660 		rc = bnx2x_alloc_mem(bp);
2661 		if (rc) {
2662 			BNX2X_ERR("Unable to allocate bp memory\n");
2663 			return rc;
2664 		}
2665 	}
2666 
2667 	/* need to be done after alloc mem, since it's self adjusting to amount
2668 	 * of memory available for RSS queues
2669 	 */
2670 	rc = bnx2x_alloc_fp_mem(bp);
2671 	if (rc) {
2672 		BNX2X_ERR("Unable to allocate memory for fps\n");
2673 		LOAD_ERROR_EXIT(bp, load_error0);
2674 	}
2675 
2676 	/* Allocated memory for FW statistics  */
2677 	rc = bnx2x_alloc_fw_stats_mem(bp);
2678 	if (rc)
2679 		LOAD_ERROR_EXIT(bp, load_error0);
2680 
2681 	/* request pf to initialize status blocks */
2682 	if (IS_VF(bp)) {
2683 		rc = bnx2x_vfpf_init(bp);
2684 		if (rc)
2685 			LOAD_ERROR_EXIT(bp, load_error0);
2686 	}
2687 
2688 	/* As long as bnx2x_alloc_mem() may possibly update
2689 	 * bp->num_queues, bnx2x_set_real_num_queues() should always
2690 	 * come after it. At this stage cnic queues are not counted.
2691 	 */
2692 	rc = bnx2x_set_real_num_queues(bp, 0);
2693 	if (rc) {
2694 		BNX2X_ERR("Unable to set real_num_queues\n");
2695 		LOAD_ERROR_EXIT(bp, load_error0);
2696 	}
2697 
2698 	/* configure multi cos mappings in kernel.
2699 	 * this configuration may be overridden by a multi class queue
2700 	 * discipline or by a dcbx negotiation result.
2701 	 */
2702 	bnx2x_setup_tc(bp->dev, bp->max_cos);
2703 
2704 	/* Add all NAPI objects */
2705 	bnx2x_add_all_napi(bp);
2706 	DP(NETIF_MSG_IFUP, "napi added\n");
2707 	bnx2x_napi_enable(bp);
2708 
2709 	if (IS_PF(bp)) {
2710 		/* set pf load just before approaching the MCP */
2711 		bnx2x_set_pf_load(bp);
2712 
2713 		/* if mcp exists send load request and analyze response */
2714 		if (!BP_NOMCP(bp)) {
2715 			/* attempt to load pf */
2716 			rc = bnx2x_nic_load_request(bp, &load_code);
2717 			if (rc)
2718 				LOAD_ERROR_EXIT(bp, load_error1);
2719 
2720 			/* what did mcp say? */
2721 			rc = bnx2x_compare_fw_ver(bp, load_code, true);
2722 			if (rc) {
2723 				bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2724 				LOAD_ERROR_EXIT(bp, load_error2);
2725 			}
2726 		} else {
2727 			load_code = bnx2x_nic_load_no_mcp(bp, port);
2728 		}
2729 
2730 		/* mark pmf if applicable */
2731 		bnx2x_nic_load_pmf(bp, load_code);
2732 
2733 		/* Init Function state controlling object */
2734 		bnx2x__init_func_obj(bp);
2735 
2736 		/* Initialize HW */
2737 		rc = bnx2x_init_hw(bp, load_code);
2738 		if (rc) {
2739 			BNX2X_ERR("HW init failed, aborting\n");
2740 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2741 			LOAD_ERROR_EXIT(bp, load_error2);
2742 		}
2743 	}
2744 
2745 	bnx2x_pre_irq_nic_init(bp);
2746 
2747 	/* Connect to IRQs */
2748 	rc = bnx2x_setup_irqs(bp);
2749 	if (rc) {
2750 		BNX2X_ERR("setup irqs failed\n");
2751 		if (IS_PF(bp))
2752 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2753 		LOAD_ERROR_EXIT(bp, load_error2);
2754 	}
2755 
2756 	/* Init per-function objects */
2757 	if (IS_PF(bp)) {
2758 		/* Setup NIC internals and enable interrupts */
2759 		bnx2x_post_irq_nic_init(bp, load_code);
2760 
2761 		bnx2x_init_bp_objs(bp);
2762 		bnx2x_iov_nic_init(bp);
2763 
2764 		/* Set AFEX default VLAN tag to an invalid value */
2765 		bp->afex_def_vlan_tag = -1;
2766 		bnx2x_nic_load_afex_dcc(bp, load_code);
2767 		bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2768 		rc = bnx2x_func_start(bp);
2769 		if (rc) {
2770 			BNX2X_ERR("Function start failed!\n");
2771 			bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2772 
2773 			LOAD_ERROR_EXIT(bp, load_error3);
2774 		}
2775 
2776 		/* Send LOAD_DONE command to MCP */
2777 		if (!BP_NOMCP(bp)) {
2778 			load_code = bnx2x_fw_command(bp,
2779 						     DRV_MSG_CODE_LOAD_DONE, 0);
2780 			if (!load_code) {
2781 				BNX2X_ERR("MCP response failure, aborting\n");
2782 				rc = -EBUSY;
2783 				LOAD_ERROR_EXIT(bp, load_error3);
2784 			}
2785 		}
2786 
2787 		/* initialize FW coalescing state machines in RAM */
2788 		bnx2x_update_coalesce(bp);
2789 	}
2790 
2791 	/* setup the leading queue */
2792 	rc = bnx2x_setup_leading(bp);
2793 	if (rc) {
2794 		BNX2X_ERR("Setup leading failed!\n");
2795 		LOAD_ERROR_EXIT(bp, load_error3);
2796 	}
2797 
2798 	/* set up the rest of the queues */
2799 	for_each_nondefault_eth_queue(bp, i) {
2800 		if (IS_PF(bp))
2801 			rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2802 		else /* VF */
2803 			rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2804 		if (rc) {
2805 			BNX2X_ERR("Queue %d setup failed\n", i);
2806 			LOAD_ERROR_EXIT(bp, load_error3);
2807 		}
2808 	}
2809 
2810 	/* setup rss */
2811 	rc = bnx2x_init_rss(bp);
2812 	if (rc) {
2813 		BNX2X_ERR("PF RSS init failed\n");
2814 		LOAD_ERROR_EXIT(bp, load_error3);
2815 	}
2816 
2817 	/* Now when Clients are configured we are ready to work */
2818 	bp->state = BNX2X_STATE_OPEN;
2819 
2820 	/* Configure a ucast MAC */
2821 	if (IS_PF(bp))
2822 		rc = bnx2x_set_eth_mac(bp, true);
2823 	else /* vf */
2824 		rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2825 					   true);
2826 	if (rc) {
2827 		BNX2X_ERR("Setting Ethernet MAC failed\n");
2828 		LOAD_ERROR_EXIT(bp, load_error3);
2829 	}
2830 
2831 	if (IS_PF(bp) && bp->pending_max) {
2832 		bnx2x_update_max_mf_config(bp, bp->pending_max);
2833 		bp->pending_max = 0;
2834 	}
2835 
2836 	bp->force_link_down = false;
2837 	if (bp->port.pmf) {
2838 		rc = bnx2x_initial_phy_init(bp, load_mode);
2839 		if (rc)
2840 			LOAD_ERROR_EXIT(bp, load_error3);
2841 	}
2842 	bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2843 
2844 	/* Start fast path */
2845 
2846 	/* Re-configure vlan filters */
2847 	rc = bnx2x_vlan_reconfigure_vid(bp);
2848 	if (rc)
2849 		LOAD_ERROR_EXIT(bp, load_error3);
2850 
2851 	/* Initialize Rx filter. */
2852 	bnx2x_set_rx_mode_inner(bp);
2853 
2854 	if (bp->flags & PTP_SUPPORTED) {
2855 		bnx2x_register_phc(bp);
2856 		bnx2x_init_ptp(bp);
2857 		bnx2x_configure_ptp_filters(bp);
2858 	}
2859 	/* Start Tx */
2860 	switch (load_mode) {
2861 	case LOAD_NORMAL:
2862 		/* Tx queue should be only re-enabled */
2863 		netif_tx_wake_all_queues(bp->dev);
2864 		break;
2865 
2866 	case LOAD_OPEN:
2867 		netif_tx_start_all_queues(bp->dev);
2868 		smp_mb__after_atomic();
2869 		break;
2870 
2871 	case LOAD_DIAG:
2872 	case LOAD_LOOPBACK_EXT:
2873 		bp->state = BNX2X_STATE_DIAG;
2874 		break;
2875 
2876 	default:
2877 		break;
2878 	}
2879 
2880 	if (bp->port.pmf)
2881 		bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2882 	else
2883 		bnx2x__link_status_update(bp);
2884 
2885 	/* start the timer */
2886 	mod_timer(&bp->timer, jiffies + bp->current_interval);
2887 
2888 	if (CNIC_ENABLED(bp))
2889 		bnx2x_load_cnic(bp);
2890 
2891 	if (IS_PF(bp))
2892 		bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2893 
2894 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2895 		/* mark driver is loaded in shmem2 */
2896 		u32 val;
2897 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2898 		val &= ~DRV_FLAGS_MTU_MASK;
2899 		val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT);
2900 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2901 			  val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2902 			  DRV_FLAGS_CAPABILITIES_LOADED_L2);
2903 	}
2904 
2905 	/* Wait for all pending SP commands to complete */
2906 	if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2907 		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2908 		bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2909 		return -EBUSY;
2910 	}
2911 
2912 	/* Update driver data for On-Chip MFW dump. */
2913 	if (IS_PF(bp))
2914 		bnx2x_update_mfw_dump(bp);
2915 
2916 	/* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2917 	if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2918 		bnx2x_dcbx_init(bp, false);
2919 
2920 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2921 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE);
2922 
2923 	DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2924 
2925 	return 0;
2926 
2927 #ifndef BNX2X_STOP_ON_ERROR
2928 load_error3:
2929 	if (IS_PF(bp)) {
2930 		bnx2x_int_disable_sync(bp, 1);
2931 
2932 		/* Clean queueable objects */
2933 		bnx2x_squeeze_objects(bp);
2934 	}
2935 
2936 	/* Free SKBs, SGEs, TPA pool and driver internals */
2937 	bnx2x_free_skbs(bp);
2938 	for_each_rx_queue(bp, i)
2939 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2940 
2941 	/* Release IRQs */
2942 	bnx2x_free_irq(bp);
2943 load_error2:
2944 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
2945 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2946 		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2947 	}
2948 
2949 	bp->port.pmf = 0;
2950 load_error1:
2951 	bnx2x_napi_disable(bp);
2952 	bnx2x_del_all_napi(bp);
2953 
2954 	/* clear pf_load status, as it was already set */
2955 	if (IS_PF(bp))
2956 		bnx2x_clear_pf_load(bp);
2957 load_error0:
2958 	bnx2x_free_fw_stats_mem(bp);
2959 	bnx2x_free_fp_mem(bp);
2960 	bnx2x_free_mem(bp);
2961 
2962 	return rc;
2963 #endif /* ! BNX2X_STOP_ON_ERROR */
2964 }
2965 
bnx2x_drain_tx_queues(struct bnx2x * bp)2966 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2967 {
2968 	u8 rc = 0, cos, i;
2969 
2970 	/* Wait until tx fastpath tasks complete */
2971 	for_each_tx_queue(bp, i) {
2972 		struct bnx2x_fastpath *fp = &bp->fp[i];
2973 
2974 		for_each_cos_in_tx_queue(fp, cos)
2975 			rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2976 		if (rc)
2977 			return rc;
2978 	}
2979 	return 0;
2980 }
2981 
2982 /* must be called with rtnl_lock */
bnx2x_nic_unload(struct bnx2x * bp,int unload_mode,bool keep_link)2983 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2984 {
2985 	int i;
2986 	bool global = false;
2987 
2988 	DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2989 
2990 	if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2991 		bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED);
2992 
2993 	/* mark driver is unloaded in shmem2 */
2994 	if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2995 		u32 val;
2996 		val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2997 		SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2998 			  val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2999 	}
3000 
3001 	if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
3002 	    (bp->state == BNX2X_STATE_CLOSED ||
3003 	     bp->state == BNX2X_STATE_ERROR)) {
3004 		/* We can get here if the driver has been unloaded
3005 		 * during parity error recovery and is either waiting for a
3006 		 * leader to complete or for other functions to unload and
3007 		 * then ifdown has been issued. In this case we want to
3008 		 * unload and let other functions to complete a recovery
3009 		 * process.
3010 		 */
3011 		bp->recovery_state = BNX2X_RECOVERY_DONE;
3012 		bp->is_leader = 0;
3013 		bnx2x_release_leader_lock(bp);
3014 		smp_mb();
3015 
3016 		DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
3017 		BNX2X_ERR("Can't unload in closed or error state\n");
3018 		return -EINVAL;
3019 	}
3020 
3021 	/* Nothing to do during unload if previous bnx2x_nic_load()
3022 	 * have not completed successfully - all resources are released.
3023 	 *
3024 	 * we can get here only after unsuccessful ndo_* callback, during which
3025 	 * dev->IFF_UP flag is still on.
3026 	 */
3027 	if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3028 		return 0;
3029 
3030 	/* It's important to set the bp->state to the value different from
3031 	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3032 	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3033 	 */
3034 	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3035 	smp_mb();
3036 
3037 	/* indicate to VFs that the PF is going down */
3038 	bnx2x_iov_channel_down(bp);
3039 
3040 	if (CNIC_LOADED(bp))
3041 		bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3042 
3043 	/* Stop Tx */
3044 	bnx2x_tx_disable(bp);
3045 	netdev_reset_tc(bp->dev);
3046 
3047 	bp->rx_mode = BNX2X_RX_MODE_NONE;
3048 
3049 	del_timer_sync(&bp->timer);
3050 
3051 	if (IS_PF(bp) && !BP_NOMCP(bp)) {
3052 		/* Set ALWAYS_ALIVE bit in shmem */
3053 		bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3054 		bnx2x_drv_pulse(bp);
3055 		bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3056 		bnx2x_save_statistics(bp);
3057 	}
3058 
3059 	/* wait till consumers catch up with producers in all queues.
3060 	 * If we're recovering, FW can't write to host so no reason
3061 	 * to wait for the queues to complete all Tx.
3062 	 */
3063 	if (unload_mode != UNLOAD_RECOVERY)
3064 		bnx2x_drain_tx_queues(bp);
3065 
3066 	/* if VF indicate to PF this function is going down (PF will delete sp
3067 	 * elements and clear initializations
3068 	 */
3069 	if (IS_VF(bp)) {
3070 		bnx2x_clear_vlan_info(bp);
3071 		bnx2x_vfpf_close_vf(bp);
3072 	} else if (unload_mode != UNLOAD_RECOVERY) {
3073 		/* if this is a normal/close unload need to clean up chip*/
3074 		bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3075 	} else {
3076 		/* Send the UNLOAD_REQUEST to the MCP */
3077 		bnx2x_send_unload_req(bp, unload_mode);
3078 
3079 		/* Prevent transactions to host from the functions on the
3080 		 * engine that doesn't reset global blocks in case of global
3081 		 * attention once global blocks are reset and gates are opened
3082 		 * (the engine which leader will perform the recovery
3083 		 * last).
3084 		 */
3085 		if (!CHIP_IS_E1x(bp))
3086 			bnx2x_pf_disable(bp);
3087 
3088 		/* Disable HW interrupts, NAPI */
3089 		bnx2x_netif_stop(bp, 1);
3090 		/* Delete all NAPI objects */
3091 		bnx2x_del_all_napi(bp);
3092 		if (CNIC_LOADED(bp))
3093 			bnx2x_del_all_napi_cnic(bp);
3094 		/* Release IRQs */
3095 		bnx2x_free_irq(bp);
3096 
3097 		/* Report UNLOAD_DONE to MCP */
3098 		bnx2x_send_unload_done(bp, false);
3099 	}
3100 
3101 	/*
3102 	 * At this stage no more interrupts will arrive so we may safely clean
3103 	 * the queueable objects here in case they failed to get cleaned so far.
3104 	 */
3105 	if (IS_PF(bp))
3106 		bnx2x_squeeze_objects(bp);
3107 
3108 	/* There should be no more pending SP commands at this stage */
3109 	bp->sp_state = 0;
3110 
3111 	bp->port.pmf = 0;
3112 
3113 	/* clear pending work in rtnl task */
3114 	bp->sp_rtnl_state = 0;
3115 	smp_mb();
3116 
3117 	/* Free SKBs, SGEs, TPA pool and driver internals */
3118 	bnx2x_free_skbs(bp);
3119 	if (CNIC_LOADED(bp))
3120 		bnx2x_free_skbs_cnic(bp);
3121 	for_each_rx_queue(bp, i)
3122 		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3123 
3124 	bnx2x_free_fp_mem(bp);
3125 	if (CNIC_LOADED(bp))
3126 		bnx2x_free_fp_mem_cnic(bp);
3127 
3128 	if (IS_PF(bp)) {
3129 		if (CNIC_LOADED(bp))
3130 			bnx2x_free_mem_cnic(bp);
3131 	}
3132 	bnx2x_free_mem(bp);
3133 
3134 	bp->state = BNX2X_STATE_CLOSED;
3135 	bp->cnic_loaded = false;
3136 
3137 	/* Clear driver version indication in shmem */
3138 	if (IS_PF(bp) && !BP_NOMCP(bp))
3139 		bnx2x_update_mng_version(bp);
3140 
3141 	/* Check if there are pending parity attentions. If there are - set
3142 	 * RECOVERY_IN_PROGRESS.
3143 	 */
3144 	if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3145 		bnx2x_set_reset_in_progress(bp);
3146 
3147 		/* Set RESET_IS_GLOBAL if needed */
3148 		if (global)
3149 			bnx2x_set_reset_global(bp);
3150 	}
3151 
3152 	/* The last driver must disable a "close the gate" if there is no
3153 	 * parity attention or "process kill" pending.
3154 	 */
3155 	if (IS_PF(bp) &&
3156 	    !bnx2x_clear_pf_load(bp) &&
3157 	    bnx2x_reset_is_done(bp, BP_PATH(bp)))
3158 		bnx2x_disable_close_the_gate(bp);
3159 
3160 	DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3161 
3162 	return 0;
3163 }
3164 
bnx2x_set_power_state(struct bnx2x * bp,pci_power_t state)3165 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3166 {
3167 	u16 pmcsr;
3168 
3169 	/* If there is no power capability, silently succeed */
3170 	if (!bp->pdev->pm_cap) {
3171 		BNX2X_DEV_INFO("No power capability. Breaking.\n");
3172 		return 0;
3173 	}
3174 
3175 	pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3176 
3177 	switch (state) {
3178 	case PCI_D0:
3179 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3180 				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3181 				       PCI_PM_CTRL_PME_STATUS));
3182 
3183 		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3184 			/* delay required during transition out of D3hot */
3185 			msleep(20);
3186 		break;
3187 
3188 	case PCI_D3hot:
3189 		/* If there are other clients above don't
3190 		   shut down the power */
3191 		if (atomic_read(&bp->pdev->enable_cnt) != 1)
3192 			return 0;
3193 		/* Don't shut down the power for emulation and FPGA */
3194 		if (CHIP_REV_IS_SLOW(bp))
3195 			return 0;
3196 
3197 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3198 		pmcsr |= 3;
3199 
3200 		if (bp->wol)
3201 			pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3202 
3203 		pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3204 				      pmcsr);
3205 
3206 		/* No more memory access after this point until
3207 		* device is brought back to D0.
3208 		*/
3209 		break;
3210 
3211 	default:
3212 		dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3213 		return -EINVAL;
3214 	}
3215 	return 0;
3216 }
3217 
3218 /*
3219  * net_device service functions
3220  */
bnx2x_poll(struct napi_struct * napi,int budget)3221 static int bnx2x_poll(struct napi_struct *napi, int budget)
3222 {
3223 	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3224 						 napi);
3225 	struct bnx2x *bp = fp->bp;
3226 	int rx_work_done;
3227 	u8 cos;
3228 
3229 #ifdef BNX2X_STOP_ON_ERROR
3230 	if (unlikely(bp->panic)) {
3231 		napi_complete(napi);
3232 		return 0;
3233 	}
3234 #endif
3235 	for_each_cos_in_tx_queue(fp, cos)
3236 		if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3237 			bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3238 
3239 	rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0;
3240 
3241 	if (rx_work_done < budget) {
3242 		/* No need to update SB for FCoE L2 ring as long as
3243 		 * it's connected to the default SB and the SB
3244 		 * has been updated when NAPI was scheduled.
3245 		 */
3246 		if (IS_FCOE_FP(fp)) {
3247 			napi_complete_done(napi, rx_work_done);
3248 		} else {
3249 			bnx2x_update_fpsb_idx(fp);
3250 			/* bnx2x_has_rx_work() reads the status block,
3251 			 * thus we need to ensure that status block indices
3252 			 * have been actually read (bnx2x_update_fpsb_idx)
3253 			 * prior to this check (bnx2x_has_rx_work) so that
3254 			 * we won't write the "newer" value of the status block
3255 			 * to IGU (if there was a DMA right after
3256 			 * bnx2x_has_rx_work and if there is no rmb, the memory
3257 			 * reading (bnx2x_update_fpsb_idx) may be postponed
3258 			 * to right before bnx2x_ack_sb). In this case there
3259 			 * will never be another interrupt until there is
3260 			 * another update of the status block, while there
3261 			 * is still unhandled work.
3262 			 */
3263 			rmb();
3264 
3265 			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3266 				if (napi_complete_done(napi, rx_work_done)) {
3267 					/* Re-enable interrupts */
3268 					DP(NETIF_MSG_RX_STATUS,
3269 					   "Update index to %d\n", fp->fp_hc_idx);
3270 					bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3271 						     le16_to_cpu(fp->fp_hc_idx),
3272 						     IGU_INT_ENABLE, 1);
3273 				}
3274 			} else {
3275 				rx_work_done = budget;
3276 			}
3277 		}
3278 	}
3279 
3280 	return rx_work_done;
3281 }
3282 
3283 /* we split the first BD into headers and data BDs
3284  * to ease the pain of our fellow microcode engineers
3285  * we use one mapping for both BDs
3286  */
bnx2x_tx_split(struct bnx2x * bp,struct bnx2x_fp_txdata * txdata,struct sw_tx_bd * tx_buf,struct eth_tx_start_bd ** tx_bd,u16 hlen,u16 bd_prod)3287 static u16 bnx2x_tx_split(struct bnx2x *bp,
3288 			  struct bnx2x_fp_txdata *txdata,
3289 			  struct sw_tx_bd *tx_buf,
3290 			  struct eth_tx_start_bd **tx_bd, u16 hlen,
3291 			  u16 bd_prod)
3292 {
3293 	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3294 	struct eth_tx_bd *d_tx_bd;
3295 	dma_addr_t mapping;
3296 	int old_len = le16_to_cpu(h_tx_bd->nbytes);
3297 
3298 	/* first fix first BD */
3299 	h_tx_bd->nbytes = cpu_to_le16(hlen);
3300 
3301 	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d (%x:%x)\n",
3302 	   h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3303 
3304 	/* now get a new data BD
3305 	 * (after the pbd) and fill it */
3306 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3307 	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3308 
3309 	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3310 			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3311 
3312 	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3313 	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3314 	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3315 
3316 	/* this marks the BD as one that has no individual mapping */
3317 	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3318 
3319 	DP(NETIF_MSG_TX_QUEUED,
3320 	   "TSO split data size is %d (%x:%x)\n",
3321 	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3322 
3323 	/* update tx_bd */
3324 	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3325 
3326 	return bd_prod;
3327 }
3328 
3329 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3330 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
bnx2x_csum_fix(unsigned char * t_header,u16 csum,s8 fix)3331 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3332 {
3333 	__sum16 tsum = (__force __sum16) csum;
3334 
3335 	if (fix > 0)
3336 		tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3337 				  csum_partial(t_header - fix, fix, 0)));
3338 
3339 	else if (fix < 0)
3340 		tsum = ~csum_fold(csum_add((__force __wsum) csum,
3341 				  csum_partial(t_header, -fix, 0)));
3342 
3343 	return bswab16(tsum);
3344 }
3345 
bnx2x_xmit_type(struct bnx2x * bp,struct sk_buff * skb)3346 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3347 {
3348 	u32 rc;
3349 	__u8 prot = 0;
3350 	__be16 protocol;
3351 
3352 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3353 		return XMIT_PLAIN;
3354 
3355 	protocol = vlan_get_protocol(skb);
3356 	if (protocol == htons(ETH_P_IPV6)) {
3357 		rc = XMIT_CSUM_V6;
3358 		prot = ipv6_hdr(skb)->nexthdr;
3359 	} else {
3360 		rc = XMIT_CSUM_V4;
3361 		prot = ip_hdr(skb)->protocol;
3362 	}
3363 
3364 	if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3365 		if (inner_ip_hdr(skb)->version == 6) {
3366 			rc |= XMIT_CSUM_ENC_V6;
3367 			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3368 				rc |= XMIT_CSUM_TCP;
3369 		} else {
3370 			rc |= XMIT_CSUM_ENC_V4;
3371 			if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3372 				rc |= XMIT_CSUM_TCP;
3373 		}
3374 	}
3375 	if (prot == IPPROTO_TCP)
3376 		rc |= XMIT_CSUM_TCP;
3377 
3378 	if (skb_is_gso(skb)) {
3379 		if (skb_is_gso_v6(skb)) {
3380 			rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3381 			if (rc & XMIT_CSUM_ENC)
3382 				rc |= XMIT_GSO_ENC_V6;
3383 		} else {
3384 			rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3385 			if (rc & XMIT_CSUM_ENC)
3386 				rc |= XMIT_GSO_ENC_V4;
3387 		}
3388 	}
3389 
3390 	return rc;
3391 }
3392 
3393 /* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */
3394 #define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS         4
3395 
3396 /* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3397 #define BNX2X_NUM_TSO_WIN_SUB_BDS               3
3398 
3399 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3400 /* check if packet requires linearization (packet is too fragmented)
3401    no need to check fragmentation if page size > 8K (there will be no
3402    violation to FW restrictions) */
bnx2x_pkt_req_lin(struct bnx2x * bp,struct sk_buff * skb,u32 xmit_type)3403 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3404 			     u32 xmit_type)
3405 {
3406 	int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS;
3407 	int to_copy = 0, hlen = 0;
3408 
3409 	if (xmit_type & XMIT_GSO_ENC)
3410 		num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS;
3411 
3412 	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) {
3413 		if (xmit_type & XMIT_GSO) {
3414 			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3415 			int wnd_size = MAX_FETCH_BD - num_tso_win_sub;
3416 			/* Number of windows to check */
3417 			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3418 			int wnd_idx = 0;
3419 			int frag_idx = 0;
3420 			u32 wnd_sum = 0;
3421 
3422 			/* Headers length */
3423 			if (xmit_type & XMIT_GSO_ENC)
3424 				hlen = (int)(skb_inner_transport_header(skb) -
3425 					     skb->data) +
3426 					     inner_tcp_hdrlen(skb);
3427 			else
3428 				hlen = (int)(skb_transport_header(skb) -
3429 					     skb->data) + tcp_hdrlen(skb);
3430 
3431 			/* Amount of data (w/o headers) on linear part of SKB*/
3432 			first_bd_sz = skb_headlen(skb) - hlen;
3433 
3434 			wnd_sum  = first_bd_sz;
3435 
3436 			/* Calculate the first sum - it's special */
3437 			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3438 				wnd_sum +=
3439 					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3440 
3441 			/* If there was data on linear skb data - check it */
3442 			if (first_bd_sz > 0) {
3443 				if (unlikely(wnd_sum < lso_mss)) {
3444 					to_copy = 1;
3445 					goto exit_lbl;
3446 				}
3447 
3448 				wnd_sum -= first_bd_sz;
3449 			}
3450 
3451 			/* Others are easier: run through the frag list and
3452 			   check all windows */
3453 			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3454 				wnd_sum +=
3455 			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3456 
3457 				if (unlikely(wnd_sum < lso_mss)) {
3458 					to_copy = 1;
3459 					break;
3460 				}
3461 				wnd_sum -=
3462 					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3463 			}
3464 		} else {
3465 			/* in non-LSO too fragmented packet should always
3466 			   be linearized */
3467 			to_copy = 1;
3468 		}
3469 	}
3470 
3471 exit_lbl:
3472 	if (unlikely(to_copy))
3473 		DP(NETIF_MSG_TX_QUEUED,
3474 		   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3475 		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3476 		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3477 
3478 	return to_copy;
3479 }
3480 #endif
3481 
3482 /**
3483  * bnx2x_set_pbd_gso - update PBD in GSO case.
3484  *
3485  * @skb:	packet skb
3486  * @pbd:	parse BD
3487  * @xmit_type:	xmit flags
3488  */
bnx2x_set_pbd_gso(struct sk_buff * skb,struct eth_tx_parse_bd_e1x * pbd,u32 xmit_type)3489 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3490 			      struct eth_tx_parse_bd_e1x *pbd,
3491 			      u32 xmit_type)
3492 {
3493 	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3494 	pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3495 	pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3496 
3497 	if (xmit_type & XMIT_GSO_V4) {
3498 		pbd->ip_id = bswab16(ip_hdr(skb)->id);
3499 		pbd->tcp_pseudo_csum =
3500 			bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3501 						   ip_hdr(skb)->daddr,
3502 						   0, IPPROTO_TCP, 0));
3503 	} else {
3504 		pbd->tcp_pseudo_csum =
3505 			bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3506 						 &ipv6_hdr(skb)->daddr,
3507 						 0, IPPROTO_TCP, 0));
3508 	}
3509 
3510 	pbd->global_data |=
3511 		cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3512 }
3513 
3514 /**
3515  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3516  *
3517  * @bp:			driver handle
3518  * @skb:		packet skb
3519  * @parsing_data:	data to be updated
3520  * @xmit_type:		xmit flags
3521  *
3522  * 57712/578xx related, when skb has encapsulation
3523  */
bnx2x_set_pbd_csum_enc(struct bnx2x * bp,struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3524 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3525 				 u32 *parsing_data, u32 xmit_type)
3526 {
3527 	*parsing_data |=
3528 		((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3529 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3530 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3531 
3532 	if (xmit_type & XMIT_CSUM_TCP) {
3533 		*parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3534 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3535 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3536 
3537 		return skb_inner_transport_header(skb) +
3538 			inner_tcp_hdrlen(skb) - skb->data;
3539 	}
3540 
3541 	/* We support checksum offload for TCP and UDP only.
3542 	 * No need to pass the UDP header length - it's a constant.
3543 	 */
3544 	return skb_inner_transport_header(skb) +
3545 		sizeof(struct udphdr) - skb->data;
3546 }
3547 
3548 /**
3549  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3550  *
3551  * @bp:			driver handle
3552  * @skb:		packet skb
3553  * @parsing_data:	data to be updated
3554  * @xmit_type:		xmit flags
3555  *
3556  * 57712/578xx related
3557  */
bnx2x_set_pbd_csum_e2(struct bnx2x * bp,struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3558 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3559 				u32 *parsing_data, u32 xmit_type)
3560 {
3561 	*parsing_data |=
3562 		((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3563 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3564 		ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3565 
3566 	if (xmit_type & XMIT_CSUM_TCP) {
3567 		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3568 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3569 			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3570 
3571 		return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3572 	}
3573 	/* We support checksum offload for TCP and UDP only.
3574 	 * No need to pass the UDP header length - it's a constant.
3575 	 */
3576 	return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3577 }
3578 
3579 /* set FW indication according to inner or outer protocols if tunneled */
bnx2x_set_sbd_csum(struct bnx2x * bp,struct sk_buff * skb,struct eth_tx_start_bd * tx_start_bd,u32 xmit_type)3580 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3581 			       struct eth_tx_start_bd *tx_start_bd,
3582 			       u32 xmit_type)
3583 {
3584 	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3585 
3586 	if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3587 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3588 
3589 	if (!(xmit_type & XMIT_CSUM_TCP))
3590 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3591 }
3592 
3593 /**
3594  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3595  *
3596  * @bp:		driver handle
3597  * @skb:	packet skb
3598  * @pbd:	parse BD to be updated
3599  * @xmit_type:	xmit flags
3600  */
bnx2x_set_pbd_csum(struct bnx2x * bp,struct sk_buff * skb,struct eth_tx_parse_bd_e1x * pbd,u32 xmit_type)3601 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3602 			     struct eth_tx_parse_bd_e1x *pbd,
3603 			     u32 xmit_type)
3604 {
3605 	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3606 
3607 	/* for now NS flag is not used in Linux */
3608 	pbd->global_data =
3609 		cpu_to_le16(hlen |
3610 			    ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3611 			     ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3612 
3613 	pbd->ip_hlen_w = (skb_transport_header(skb) -
3614 			skb_network_header(skb)) >> 1;
3615 
3616 	hlen += pbd->ip_hlen_w;
3617 
3618 	/* We support checksum offload for TCP and UDP only */
3619 	if (xmit_type & XMIT_CSUM_TCP)
3620 		hlen += tcp_hdrlen(skb) / 2;
3621 	else
3622 		hlen += sizeof(struct udphdr) / 2;
3623 
3624 	pbd->total_hlen_w = cpu_to_le16(hlen);
3625 	hlen = hlen*2;
3626 
3627 	if (xmit_type & XMIT_CSUM_TCP) {
3628 		pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3629 
3630 	} else {
3631 		s8 fix = SKB_CS_OFF(skb); /* signed! */
3632 
3633 		DP(NETIF_MSG_TX_QUEUED,
3634 		   "hlen %d  fix %d  csum before fix %x\n",
3635 		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3636 
3637 		/* HW bug: fixup the CSUM */
3638 		pbd->tcp_pseudo_csum =
3639 			bnx2x_csum_fix(skb_transport_header(skb),
3640 				       SKB_CS(skb), fix);
3641 
3642 		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3643 		   pbd->tcp_pseudo_csum);
3644 	}
3645 
3646 	return hlen;
3647 }
3648 
bnx2x_update_pbds_gso_enc(struct sk_buff * skb,struct eth_tx_parse_bd_e2 * pbd_e2,struct eth_tx_parse_2nd_bd * pbd2,u16 * global_data,u32 xmit_type)3649 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3650 				      struct eth_tx_parse_bd_e2 *pbd_e2,
3651 				      struct eth_tx_parse_2nd_bd *pbd2,
3652 				      u16 *global_data,
3653 				      u32 xmit_type)
3654 {
3655 	u16 hlen_w = 0;
3656 	u8 outerip_off, outerip_len = 0;
3657 
3658 	/* from outer IP to transport */
3659 	hlen_w = (skb_inner_transport_header(skb) -
3660 		  skb_network_header(skb)) >> 1;
3661 
3662 	/* transport len */
3663 	hlen_w += inner_tcp_hdrlen(skb) >> 1;
3664 
3665 	pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3666 
3667 	/* outer IP header info */
3668 	if (xmit_type & XMIT_CSUM_V4) {
3669 		struct iphdr *iph = ip_hdr(skb);
3670 		u32 csum = (__force u32)(~iph->check) -
3671 			   (__force u32)iph->tot_len -
3672 			   (__force u32)iph->frag_off;
3673 
3674 		outerip_len = iph->ihl << 1;
3675 
3676 		pbd2->fw_ip_csum_wo_len_flags_frag =
3677 			bswab16(csum_fold((__force __wsum)csum));
3678 	} else {
3679 		pbd2->fw_ip_hdr_to_payload_w =
3680 			hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3681 		pbd_e2->data.tunnel_data.flags |=
3682 			ETH_TUNNEL_DATA_IPV6_OUTER;
3683 	}
3684 
3685 	pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3686 
3687 	pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3688 
3689 	/* inner IP header info */
3690 	if (xmit_type & XMIT_CSUM_ENC_V4) {
3691 		pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3692 
3693 		pbd_e2->data.tunnel_data.pseudo_csum =
3694 			bswab16(~csum_tcpudp_magic(
3695 					inner_ip_hdr(skb)->saddr,
3696 					inner_ip_hdr(skb)->daddr,
3697 					0, IPPROTO_TCP, 0));
3698 	} else {
3699 		pbd_e2->data.tunnel_data.pseudo_csum =
3700 			bswab16(~csum_ipv6_magic(
3701 					&inner_ipv6_hdr(skb)->saddr,
3702 					&inner_ipv6_hdr(skb)->daddr,
3703 					0, IPPROTO_TCP, 0));
3704 	}
3705 
3706 	outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3707 
3708 	*global_data |=
3709 		outerip_off |
3710 		(outerip_len <<
3711 			ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3712 		((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3713 			ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3714 
3715 	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3716 		SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3717 		pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3718 	}
3719 }
3720 
bnx2x_set_ipv6_ext_e2(struct sk_buff * skb,u32 * parsing_data,u32 xmit_type)3721 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3722 					 u32 xmit_type)
3723 {
3724 	struct ipv6hdr *ipv6;
3725 
3726 	if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3727 		return;
3728 
3729 	if (xmit_type & XMIT_GSO_ENC_V6)
3730 		ipv6 = inner_ipv6_hdr(skb);
3731 	else /* XMIT_GSO_V6 */
3732 		ipv6 = ipv6_hdr(skb);
3733 
3734 	if (ipv6->nexthdr == NEXTHDR_IPV6)
3735 		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3736 }
3737 
3738 /* called with netif_tx_lock
3739  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3740  * netif_wake_queue()
3741  */
bnx2x_start_xmit(struct sk_buff * skb,struct net_device * dev)3742 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3743 {
3744 	struct bnx2x *bp = netdev_priv(dev);
3745 
3746 	struct netdev_queue *txq;
3747 	struct bnx2x_fp_txdata *txdata;
3748 	struct sw_tx_bd *tx_buf;
3749 	struct eth_tx_start_bd *tx_start_bd, *first_bd;
3750 	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3751 	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3752 	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3753 	struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3754 	u32 pbd_e2_parsing_data = 0;
3755 	u16 pkt_prod, bd_prod;
3756 	int nbd, txq_index;
3757 	dma_addr_t mapping;
3758 	u32 xmit_type = bnx2x_xmit_type(bp, skb);
3759 	int i;
3760 	u8 hlen = 0;
3761 	__le16 pkt_size = 0;
3762 	struct ethhdr *eth;
3763 	u8 mac_type = UNICAST_ADDRESS;
3764 
3765 #ifdef BNX2X_STOP_ON_ERROR
3766 	if (unlikely(bp->panic))
3767 		return NETDEV_TX_BUSY;
3768 #endif
3769 
3770 	txq_index = skb_get_queue_mapping(skb);
3771 	txq = netdev_get_tx_queue(dev, txq_index);
3772 
3773 	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3774 
3775 	txdata = &bp->bnx2x_txq[txq_index];
3776 
3777 	/* enable this debug print to view the transmission queue being used
3778 	DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3779 	   txq_index, fp_index, txdata_index); */
3780 
3781 	/* enable this debug print to view the transmission details
3782 	DP(NETIF_MSG_TX_QUEUED,
3783 	   "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3784 	   txdata->cid, fp_index, txdata_index, txdata, fp); */
3785 
3786 	if (unlikely(bnx2x_tx_avail(bp, txdata) <
3787 			skb_shinfo(skb)->nr_frags +
3788 			BDS_PER_TX_PKT +
3789 			NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3790 		/* Handle special storage cases separately */
3791 		if (txdata->tx_ring_size == 0) {
3792 			struct bnx2x_eth_q_stats *q_stats =
3793 				bnx2x_fp_qstats(bp, txdata->parent_fp);
3794 			q_stats->driver_filtered_tx_pkt++;
3795 			dev_kfree_skb(skb);
3796 			return NETDEV_TX_OK;
3797 		}
3798 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3799 		netif_tx_stop_queue(txq);
3800 		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3801 
3802 		return NETDEV_TX_BUSY;
3803 	}
3804 
3805 	DP(NETIF_MSG_TX_QUEUED,
3806 	   "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3807 	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3808 	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3809 	   skb->len);
3810 
3811 	eth = (struct ethhdr *)skb->data;
3812 
3813 	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
3814 	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3815 		if (is_broadcast_ether_addr(eth->h_dest))
3816 			mac_type = BROADCAST_ADDRESS;
3817 		else
3818 			mac_type = MULTICAST_ADDRESS;
3819 	}
3820 
3821 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3822 	/* First, check if we need to linearize the skb (due to FW
3823 	   restrictions). No need to check fragmentation if page size > 8K
3824 	   (there will be no violation to FW restrictions) */
3825 	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3826 		/* Statistics of linearization */
3827 		bp->lin_cnt++;
3828 		if (skb_linearize(skb) != 0) {
3829 			DP(NETIF_MSG_TX_QUEUED,
3830 			   "SKB linearization failed - silently dropping this SKB\n");
3831 			dev_kfree_skb_any(skb);
3832 			return NETDEV_TX_OK;
3833 		}
3834 	}
3835 #endif
3836 	/* Map skb linear data for DMA */
3837 	mapping = dma_map_single(&bp->pdev->dev, skb->data,
3838 				 skb_headlen(skb), DMA_TO_DEVICE);
3839 	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3840 		DP(NETIF_MSG_TX_QUEUED,
3841 		   "SKB mapping failed - silently dropping this SKB\n");
3842 		dev_kfree_skb_any(skb);
3843 		return NETDEV_TX_OK;
3844 	}
3845 	/*
3846 	Please read carefully. First we use one BD which we mark as start,
3847 	then we have a parsing info BD (used for TSO or xsum),
3848 	and only then we have the rest of the TSO BDs.
3849 	(don't forget to mark the last one as last,
3850 	and to unmap only AFTER you write to the BD ...)
3851 	And above all, all pdb sizes are in words - NOT DWORDS!
3852 	*/
3853 
3854 	/* get current pkt produced now - advance it just before sending packet
3855 	 * since mapping of pages may fail and cause packet to be dropped
3856 	 */
3857 	pkt_prod = txdata->tx_pkt_prod;
3858 	bd_prod = TX_BD(txdata->tx_bd_prod);
3859 
3860 	/* get a tx_buf and first BD
3861 	 * tx_start_bd may be changed during SPLIT,
3862 	 * but first_bd will always stay first
3863 	 */
3864 	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3865 	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3866 	first_bd = tx_start_bd;
3867 
3868 	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3869 
3870 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3871 		if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3872 			bp->eth_stats.ptp_skip_tx_ts++;
3873 			BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3874 		} else if (bp->ptp_tx_skb) {
3875 			bp->eth_stats.ptp_skip_tx_ts++;
3876 			netdev_err_once(bp->dev,
3877 					"Device supports only a single outstanding packet to timestamp, this packet won't be timestamped\n");
3878 		} else {
3879 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3880 			/* schedule check for Tx timestamp */
3881 			bp->ptp_tx_skb = skb_get(skb);
3882 			bp->ptp_tx_start = jiffies;
3883 			schedule_work(&bp->ptp_task);
3884 		}
3885 	}
3886 
3887 	/* header nbd: indirectly zero other flags! */
3888 	tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3889 
3890 	/* remember the first BD of the packet */
3891 	tx_buf->first_bd = txdata->tx_bd_prod;
3892 	tx_buf->skb = skb;
3893 	tx_buf->flags = 0;
3894 
3895 	DP(NETIF_MSG_TX_QUEUED,
3896 	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3897 	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3898 
3899 	if (skb_vlan_tag_present(skb)) {
3900 		tx_start_bd->vlan_or_ethertype =
3901 		    cpu_to_le16(skb_vlan_tag_get(skb));
3902 		tx_start_bd->bd_flags.as_bitfield |=
3903 		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3904 	} else {
3905 		/* when transmitting in a vf, start bd must hold the ethertype
3906 		 * for fw to enforce it
3907 		 */
3908 		u16 vlan_tci = 0;
3909 #ifndef BNX2X_STOP_ON_ERROR
3910 		if (IS_VF(bp)) {
3911 #endif
3912 			/* Still need to consider inband vlan for enforced */
3913 			if (__vlan_get_tag(skb, &vlan_tci)) {
3914 				tx_start_bd->vlan_or_ethertype =
3915 					cpu_to_le16(ntohs(eth->h_proto));
3916 			} else {
3917 				tx_start_bd->bd_flags.as_bitfield |=
3918 					(X_ETH_INBAND_VLAN <<
3919 					 ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3920 				tx_start_bd->vlan_or_ethertype =
3921 					cpu_to_le16(vlan_tci);
3922 			}
3923 #ifndef BNX2X_STOP_ON_ERROR
3924 		} else {
3925 			/* used by FW for packet accounting */
3926 			tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3927 		}
3928 #endif
3929 	}
3930 
3931 	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3932 
3933 	/* turn on parsing and get a BD */
3934 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3935 
3936 	if (xmit_type & XMIT_CSUM)
3937 		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3938 
3939 	if (!CHIP_IS_E1x(bp)) {
3940 		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3941 		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3942 
3943 		if (xmit_type & XMIT_CSUM_ENC) {
3944 			u16 global_data = 0;
3945 
3946 			/* Set PBD in enc checksum offload case */
3947 			hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3948 						      &pbd_e2_parsing_data,
3949 						      xmit_type);
3950 
3951 			/* turn on 2nd parsing and get a BD */
3952 			bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3953 
3954 			pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3955 
3956 			memset(pbd2, 0, sizeof(*pbd2));
3957 
3958 			pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3959 				(skb_inner_network_header(skb) -
3960 				 skb->data) >> 1;
3961 
3962 			if (xmit_type & XMIT_GSO_ENC)
3963 				bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3964 							  &global_data,
3965 							  xmit_type);
3966 
3967 			pbd2->global_data = cpu_to_le16(global_data);
3968 
3969 			/* add addition parse BD indication to start BD */
3970 			SET_FLAG(tx_start_bd->general_data,
3971 				 ETH_TX_START_BD_PARSE_NBDS, 1);
3972 			/* set encapsulation flag in start BD */
3973 			SET_FLAG(tx_start_bd->general_data,
3974 				 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3975 
3976 			tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3977 
3978 			nbd++;
3979 		} else if (xmit_type & XMIT_CSUM) {
3980 			/* Set PBD in checksum offload case w/o encapsulation */
3981 			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3982 						     &pbd_e2_parsing_data,
3983 						     xmit_type);
3984 		}
3985 
3986 		bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3987 		/* Add the macs to the parsing BD if this is a vf or if
3988 		 * Tx Switching is enabled.
3989 		 */
3990 		if (IS_VF(bp)) {
3991 			/* override GRE parameters in BD */
3992 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3993 					      &pbd_e2->data.mac_addr.src_mid,
3994 					      &pbd_e2->data.mac_addr.src_lo,
3995 					      eth->h_source);
3996 
3997 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3998 					      &pbd_e2->data.mac_addr.dst_mid,
3999 					      &pbd_e2->data.mac_addr.dst_lo,
4000 					      eth->h_dest);
4001 		} else {
4002 			if (bp->flags & TX_SWITCHING)
4003 				bnx2x_set_fw_mac_addr(
4004 						&pbd_e2->data.mac_addr.dst_hi,
4005 						&pbd_e2->data.mac_addr.dst_mid,
4006 						&pbd_e2->data.mac_addr.dst_lo,
4007 						eth->h_dest);
4008 #ifdef BNX2X_STOP_ON_ERROR
4009 			/* Enforce security is always set in Stop on Error -
4010 			 * source mac should be present in the parsing BD
4011 			 */
4012 			bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
4013 					      &pbd_e2->data.mac_addr.src_mid,
4014 					      &pbd_e2->data.mac_addr.src_lo,
4015 					      eth->h_source);
4016 #endif
4017 		}
4018 
4019 		SET_FLAG(pbd_e2_parsing_data,
4020 			 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
4021 	} else {
4022 		u16 global_data = 0;
4023 		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4024 		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4025 		/* Set PBD in checksum offload case */
4026 		if (xmit_type & XMIT_CSUM)
4027 			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4028 
4029 		SET_FLAG(global_data,
4030 			 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4031 		pbd_e1x->global_data |= cpu_to_le16(global_data);
4032 	}
4033 
4034 	/* Setup the data pointer of the first BD of the packet */
4035 	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4036 	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4037 	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4038 	pkt_size = tx_start_bd->nbytes;
4039 
4040 	DP(NETIF_MSG_TX_QUEUED,
4041 	   "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4042 	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4043 	   le16_to_cpu(tx_start_bd->nbytes),
4044 	   tx_start_bd->bd_flags.as_bitfield,
4045 	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4046 
4047 	if (xmit_type & XMIT_GSO) {
4048 
4049 		DP(NETIF_MSG_TX_QUEUED,
4050 		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4051 		   skb->len, hlen, skb_headlen(skb),
4052 		   skb_shinfo(skb)->gso_size);
4053 
4054 		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4055 
4056 		if (unlikely(skb_headlen(skb) > hlen)) {
4057 			nbd++;
4058 			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4059 						 &tx_start_bd, hlen,
4060 						 bd_prod);
4061 		}
4062 		if (!CHIP_IS_E1x(bp))
4063 			pbd_e2_parsing_data |=
4064 				(skb_shinfo(skb)->gso_size <<
4065 				 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4066 				 ETH_TX_PARSE_BD_E2_LSO_MSS;
4067 		else
4068 			bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4069 	}
4070 
4071 	/* Set the PBD's parsing_data field if not zero
4072 	 * (for the chips newer than 57711).
4073 	 */
4074 	if (pbd_e2_parsing_data)
4075 		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4076 
4077 	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4078 
4079 	/* Handle fragmented skb */
4080 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4081 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4082 
4083 		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4084 					   skb_frag_size(frag), DMA_TO_DEVICE);
4085 		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4086 			unsigned int pkts_compl = 0, bytes_compl = 0;
4087 
4088 			DP(NETIF_MSG_TX_QUEUED,
4089 			   "Unable to map page - dropping packet...\n");
4090 
4091 			/* we need unmap all buffers already mapped
4092 			 * for this SKB;
4093 			 * first_bd->nbd need to be properly updated
4094 			 * before call to bnx2x_free_tx_pkt
4095 			 */
4096 			first_bd->nbd = cpu_to_le16(nbd);
4097 			bnx2x_free_tx_pkt(bp, txdata,
4098 					  TX_BD(txdata->tx_pkt_prod),
4099 					  &pkts_compl, &bytes_compl);
4100 			return NETDEV_TX_OK;
4101 		}
4102 
4103 		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4104 		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4105 		if (total_pkt_bd == NULL)
4106 			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4107 
4108 		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4109 		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4110 		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4111 		le16_add_cpu(&pkt_size, skb_frag_size(frag));
4112 		nbd++;
4113 
4114 		DP(NETIF_MSG_TX_QUEUED,
4115 		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4116 		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4117 		   le16_to_cpu(tx_data_bd->nbytes));
4118 	}
4119 
4120 	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4121 
4122 	/* update with actual num BDs */
4123 	first_bd->nbd = cpu_to_le16(nbd);
4124 
4125 	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4126 
4127 	/* now send a tx doorbell, counting the next BD
4128 	 * if the packet contains or ends with it
4129 	 */
4130 	if (TX_BD_POFF(bd_prod) < nbd)
4131 		nbd++;
4132 
4133 	/* total_pkt_bytes should be set on the first data BD if
4134 	 * it's not an LSO packet and there is more than one
4135 	 * data BD. In this case pkt_size is limited by an MTU value.
4136 	 * However we prefer to set it for an LSO packet (while we don't
4137 	 * have to) in order to save some CPU cycles in a none-LSO
4138 	 * case, when we much more care about them.
4139 	 */
4140 	if (total_pkt_bd != NULL)
4141 		total_pkt_bd->total_pkt_bytes = pkt_size;
4142 
4143 	if (pbd_e1x)
4144 		DP(NETIF_MSG_TX_QUEUED,
4145 		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4146 		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4147 		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4148 		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4149 		    le16_to_cpu(pbd_e1x->total_hlen_w));
4150 	if (pbd_e2)
4151 		DP(NETIF_MSG_TX_QUEUED,
4152 		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4153 		   pbd_e2,
4154 		   pbd_e2->data.mac_addr.dst_hi,
4155 		   pbd_e2->data.mac_addr.dst_mid,
4156 		   pbd_e2->data.mac_addr.dst_lo,
4157 		   pbd_e2->data.mac_addr.src_hi,
4158 		   pbd_e2->data.mac_addr.src_mid,
4159 		   pbd_e2->data.mac_addr.src_lo,
4160 		   pbd_e2->parsing_data);
4161 	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4162 
4163 	netdev_tx_sent_queue(txq, skb->len);
4164 
4165 	skb_tx_timestamp(skb);
4166 
4167 	txdata->tx_pkt_prod++;
4168 	/*
4169 	 * Make sure that the BD data is updated before updating the producer
4170 	 * since FW might read the BD right after the producer is updated.
4171 	 * This is only applicable for weak-ordered memory model archs such
4172 	 * as IA-64. The following barrier is also mandatory since FW will
4173 	 * assumes packets must have BDs.
4174 	 */
4175 	wmb();
4176 
4177 	txdata->tx_db.data.prod += nbd;
4178 	/* make sure descriptor update is observed by HW */
4179 	wmb();
4180 
4181 	DOORBELL_RELAXED(bp, txdata->cid, txdata->tx_db.raw);
4182 
4183 	txdata->tx_bd_prod += nbd;
4184 
4185 	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4186 		netif_tx_stop_queue(txq);
4187 
4188 		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
4189 		 * ordering of set_bit() in netif_tx_stop_queue() and read of
4190 		 * fp->bd_tx_cons */
4191 		smp_mb();
4192 
4193 		bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4194 		if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4195 			netif_tx_wake_queue(txq);
4196 	}
4197 	txdata->tx_pkt++;
4198 
4199 	return NETDEV_TX_OK;
4200 }
4201 
bnx2x_get_c2s_mapping(struct bnx2x * bp,u8 * c2s_map,u8 * c2s_default)4202 void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default)
4203 {
4204 	int mfw_vn = BP_FW_MB_IDX(bp);
4205 	u32 tmp;
4206 
4207 	/* If the shmem shouldn't affect configuration, reflect */
4208 	if (!IS_MF_BD(bp)) {
4209 		int i;
4210 
4211 		for (i = 0; i < BNX2X_MAX_PRIORITY; i++)
4212 			c2s_map[i] = i;
4213 		*c2s_default = 0;
4214 
4215 		return;
4216 	}
4217 
4218 	tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]);
4219 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4220 	c2s_map[0] = tmp & 0xff;
4221 	c2s_map[1] = (tmp >> 8) & 0xff;
4222 	c2s_map[2] = (tmp >> 16) & 0xff;
4223 	c2s_map[3] = (tmp >> 24) & 0xff;
4224 
4225 	tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]);
4226 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4227 	c2s_map[4] = tmp & 0xff;
4228 	c2s_map[5] = (tmp >> 8) & 0xff;
4229 	c2s_map[6] = (tmp >> 16) & 0xff;
4230 	c2s_map[7] = (tmp >> 24) & 0xff;
4231 
4232 	tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]);
4233 	tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4234 	*c2s_default = (tmp >> (8 * mfw_vn)) & 0xff;
4235 }
4236 
4237 /**
4238  * bnx2x_setup_tc - routine to configure net_device for multi tc
4239  *
4240  * @dev: net device to configure
4241  * @num_tc: number of traffic classes to enable
4242  *
4243  * callback connected to the ndo_setup_tc function pointer
4244  */
bnx2x_setup_tc(struct net_device * dev,u8 num_tc)4245 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4246 {
4247 	struct bnx2x *bp = netdev_priv(dev);
4248 	u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def;
4249 	int cos, prio, count, offset;
4250 
4251 	/* setup tc must be called under rtnl lock */
4252 	ASSERT_RTNL();
4253 
4254 	/* no traffic classes requested. Aborting */
4255 	if (!num_tc) {
4256 		netdev_reset_tc(dev);
4257 		return 0;
4258 	}
4259 
4260 	/* requested to support too many traffic classes */
4261 	if (num_tc > bp->max_cos) {
4262 		BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4263 			  num_tc, bp->max_cos);
4264 		return -EINVAL;
4265 	}
4266 
4267 	/* declare amount of supported traffic classes */
4268 	if (netdev_set_num_tc(dev, num_tc)) {
4269 		BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4270 		return -EINVAL;
4271 	}
4272 
4273 	bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def);
4274 
4275 	/* configure priority to traffic class mapping */
4276 	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4277 		int outer_prio = c2s_map[prio];
4278 
4279 		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]);
4280 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4281 		   "mapping priority %d to tc %d\n",
4282 		   outer_prio, bp->prio_to_cos[outer_prio]);
4283 	}
4284 
4285 	/* Use this configuration to differentiate tc0 from other COSes
4286 	   This can be used for ets or pfc, and save the effort of setting
4287 	   up a multio class queue disc or negotiating DCBX with a switch
4288 	netdev_set_prio_tc_map(dev, 0, 0);
4289 	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4290 	for (prio = 1; prio < 16; prio++) {
4291 		netdev_set_prio_tc_map(dev, prio, 1);
4292 		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4293 	} */
4294 
4295 	/* configure traffic class to transmission queue mapping */
4296 	for (cos = 0; cos < bp->max_cos; cos++) {
4297 		count = BNX2X_NUM_ETH_QUEUES(bp);
4298 		offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4299 		netdev_set_tc_queue(dev, cos, count, offset);
4300 		DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4301 		   "mapping tc %d to offset %d count %d\n",
4302 		   cos, offset, count);
4303 	}
4304 
4305 	return 0;
4306 }
4307 
__bnx2x_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)4308 int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
4309 		     void *type_data)
4310 {
4311 	struct tc_mqprio_qopt *mqprio = type_data;
4312 
4313 	if (type != TC_SETUP_QDISC_MQPRIO)
4314 		return -EOPNOTSUPP;
4315 
4316 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
4317 
4318 	return bnx2x_setup_tc(dev, mqprio->num_tc);
4319 }
4320 
4321 /* called with rtnl_lock */
bnx2x_change_mac_addr(struct net_device * dev,void * p)4322 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4323 {
4324 	struct sockaddr *addr = p;
4325 	struct bnx2x *bp = netdev_priv(dev);
4326 	int rc = 0;
4327 
4328 	if (!is_valid_ether_addr(addr->sa_data)) {
4329 		BNX2X_ERR("Requested MAC address is not valid\n");
4330 		return -EINVAL;
4331 	}
4332 
4333 	if (IS_MF_STORAGE_ONLY(bp)) {
4334 		BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4335 		return -EINVAL;
4336 	}
4337 
4338 	if (netif_running(dev))  {
4339 		rc = bnx2x_set_eth_mac(bp, false);
4340 		if (rc)
4341 			return rc;
4342 	}
4343 
4344 	eth_hw_addr_set(dev, addr->sa_data);
4345 
4346 	if (netif_running(dev))
4347 		rc = bnx2x_set_eth_mac(bp, true);
4348 
4349 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4350 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4351 
4352 	return rc;
4353 }
4354 
bnx2x_free_fp_mem_at(struct bnx2x * bp,int fp_index)4355 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4356 {
4357 	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4358 	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4359 	u8 cos;
4360 
4361 	/* Common */
4362 
4363 	if (IS_FCOE_IDX(fp_index)) {
4364 		memset(sb, 0, sizeof(union host_hc_status_block));
4365 		fp->status_blk_mapping = 0;
4366 	} else {
4367 		/* status blocks */
4368 		if (!CHIP_IS_E1x(bp))
4369 			BNX2X_PCI_FREE(sb->e2_sb,
4370 				       bnx2x_fp(bp, fp_index,
4371 						status_blk_mapping),
4372 				       sizeof(struct host_hc_status_block_e2));
4373 		else
4374 			BNX2X_PCI_FREE(sb->e1x_sb,
4375 				       bnx2x_fp(bp, fp_index,
4376 						status_blk_mapping),
4377 				       sizeof(struct host_hc_status_block_e1x));
4378 	}
4379 
4380 	/* Rx */
4381 	if (!skip_rx_queue(bp, fp_index)) {
4382 		bnx2x_free_rx_bds(fp);
4383 
4384 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4385 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4386 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4387 			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
4388 			       sizeof(struct eth_rx_bd) * NUM_RX_BD);
4389 
4390 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4391 			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
4392 			       sizeof(struct eth_fast_path_rx_cqe) *
4393 			       NUM_RCQ_BD);
4394 
4395 		/* SGE ring */
4396 		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4397 		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4398 			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
4399 			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4400 	}
4401 
4402 	/* Tx */
4403 	if (!skip_tx_queue(bp, fp_index)) {
4404 		/* fastpath tx rings: tx_buf tx_desc */
4405 		for_each_cos_in_tx_queue(fp, cos) {
4406 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4407 
4408 			DP(NETIF_MSG_IFDOWN,
4409 			   "freeing tx memory of fp %d cos %d cid %d\n",
4410 			   fp_index, cos, txdata->cid);
4411 
4412 			BNX2X_FREE(txdata->tx_buf_ring);
4413 			BNX2X_PCI_FREE(txdata->tx_desc_ring,
4414 				txdata->tx_desc_mapping,
4415 				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4416 		}
4417 	}
4418 	/* end of fastpath */
4419 }
4420 
bnx2x_free_fp_mem_cnic(struct bnx2x * bp)4421 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4422 {
4423 	int i;
4424 	for_each_cnic_queue(bp, i)
4425 		bnx2x_free_fp_mem_at(bp, i);
4426 }
4427 
bnx2x_free_fp_mem(struct bnx2x * bp)4428 void bnx2x_free_fp_mem(struct bnx2x *bp)
4429 {
4430 	int i;
4431 	for_each_eth_queue(bp, i)
4432 		bnx2x_free_fp_mem_at(bp, i);
4433 }
4434 
set_sb_shortcuts(struct bnx2x * bp,int index)4435 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4436 {
4437 	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4438 	if (!CHIP_IS_E1x(bp)) {
4439 		bnx2x_fp(bp, index, sb_index_values) =
4440 			(__le16 *)status_blk.e2_sb->sb.index_values;
4441 		bnx2x_fp(bp, index, sb_running_index) =
4442 			(__le16 *)status_blk.e2_sb->sb.running_index;
4443 	} else {
4444 		bnx2x_fp(bp, index, sb_index_values) =
4445 			(__le16 *)status_blk.e1x_sb->sb.index_values;
4446 		bnx2x_fp(bp, index, sb_running_index) =
4447 			(__le16 *)status_blk.e1x_sb->sb.running_index;
4448 	}
4449 }
4450 
4451 /* Returns the number of actually allocated BDs */
bnx2x_alloc_rx_bds(struct bnx2x_fastpath * fp,int rx_ring_size)4452 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4453 			      int rx_ring_size)
4454 {
4455 	struct bnx2x *bp = fp->bp;
4456 	u16 ring_prod, cqe_ring_prod;
4457 	int i, failure_cnt = 0;
4458 
4459 	fp->rx_comp_cons = 0;
4460 	cqe_ring_prod = ring_prod = 0;
4461 
4462 	/* This routine is called only during fo init so
4463 	 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4464 	 */
4465 	for (i = 0; i < rx_ring_size; i++) {
4466 		if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4467 			failure_cnt++;
4468 			continue;
4469 		}
4470 		ring_prod = NEXT_RX_IDX(ring_prod);
4471 		cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4472 		WARN_ON(ring_prod <= (i - failure_cnt));
4473 	}
4474 
4475 	if (failure_cnt)
4476 		BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4477 			  i - failure_cnt, fp->index);
4478 
4479 	fp->rx_bd_prod = ring_prod;
4480 	/* Limit the CQE producer by the CQE ring size */
4481 	fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4482 			       cqe_ring_prod);
4483 
4484 	bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4485 
4486 	return i - failure_cnt;
4487 }
4488 
bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath * fp)4489 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4490 {
4491 	int i;
4492 
4493 	for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4494 		struct eth_rx_cqe_next_page *nextpg;
4495 
4496 		nextpg = (struct eth_rx_cqe_next_page *)
4497 			&fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4498 		nextpg->addr_hi =
4499 			cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4500 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4501 		nextpg->addr_lo =
4502 			cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4503 				   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4504 	}
4505 }
4506 
bnx2x_alloc_fp_mem_at(struct bnx2x * bp,int index)4507 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4508 {
4509 	union host_hc_status_block *sb;
4510 	struct bnx2x_fastpath *fp = &bp->fp[index];
4511 	int ring_size = 0;
4512 	u8 cos;
4513 	int rx_ring_size = 0;
4514 
4515 	if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4516 		rx_ring_size = MIN_RX_SIZE_NONTPA;
4517 		bp->rx_ring_size = rx_ring_size;
4518 	} else if (!bp->rx_ring_size) {
4519 		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4520 
4521 		if (CHIP_IS_E3(bp)) {
4522 			u32 cfg = SHMEM_RD(bp,
4523 					   dev_info.port_hw_config[BP_PORT(bp)].
4524 					   default_cfg);
4525 
4526 			/* Decrease ring size for 1G functions */
4527 			if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4528 			    PORT_HW_CFG_NET_SERDES_IF_SGMII)
4529 				rx_ring_size /= 10;
4530 		}
4531 
4532 		/* allocate at least number of buffers required by FW */
4533 		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4534 				     MIN_RX_SIZE_TPA, rx_ring_size);
4535 
4536 		bp->rx_ring_size = rx_ring_size;
4537 	} else /* if rx_ring_size specified - use it */
4538 		rx_ring_size = bp->rx_ring_size;
4539 
4540 	DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4541 
4542 	/* Common */
4543 	sb = &bnx2x_fp(bp, index, status_blk);
4544 
4545 	if (!IS_FCOE_IDX(index)) {
4546 		/* status blocks */
4547 		if (!CHIP_IS_E1x(bp)) {
4548 			sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4549 						    sizeof(struct host_hc_status_block_e2));
4550 			if (!sb->e2_sb)
4551 				goto alloc_mem_err;
4552 		} else {
4553 			sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4554 						     sizeof(struct host_hc_status_block_e1x));
4555 			if (!sb->e1x_sb)
4556 				goto alloc_mem_err;
4557 		}
4558 	}
4559 
4560 	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4561 	 * set shortcuts for it.
4562 	 */
4563 	if (!IS_FCOE_IDX(index))
4564 		set_sb_shortcuts(bp, index);
4565 
4566 	/* Tx */
4567 	if (!skip_tx_queue(bp, index)) {
4568 		/* fastpath tx rings: tx_buf tx_desc */
4569 		for_each_cos_in_tx_queue(fp, cos) {
4570 			struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4571 
4572 			DP(NETIF_MSG_IFUP,
4573 			   "allocating tx memory of fp %d cos %d\n",
4574 			   index, cos);
4575 
4576 			txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4577 						      sizeof(struct sw_tx_bd),
4578 						      GFP_KERNEL);
4579 			if (!txdata->tx_buf_ring)
4580 				goto alloc_mem_err;
4581 			txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4582 							       sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4583 			if (!txdata->tx_desc_ring)
4584 				goto alloc_mem_err;
4585 		}
4586 	}
4587 
4588 	/* Rx */
4589 	if (!skip_rx_queue(bp, index)) {
4590 		/* fastpath rx rings: rx_buf rx_desc rx_comp */
4591 		bnx2x_fp(bp, index, rx_buf_ring) =
4592 			kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4593 		if (!bnx2x_fp(bp, index, rx_buf_ring))
4594 			goto alloc_mem_err;
4595 		bnx2x_fp(bp, index, rx_desc_ring) =
4596 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4597 					sizeof(struct eth_rx_bd) * NUM_RX_BD);
4598 		if (!bnx2x_fp(bp, index, rx_desc_ring))
4599 			goto alloc_mem_err;
4600 
4601 		/* Seed all CQEs by 1s */
4602 		bnx2x_fp(bp, index, rx_comp_ring) =
4603 			BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4604 					 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4605 		if (!bnx2x_fp(bp, index, rx_comp_ring))
4606 			goto alloc_mem_err;
4607 
4608 		/* SGE ring */
4609 		bnx2x_fp(bp, index, rx_page_ring) =
4610 			kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4611 				GFP_KERNEL);
4612 		if (!bnx2x_fp(bp, index, rx_page_ring))
4613 			goto alloc_mem_err;
4614 		bnx2x_fp(bp, index, rx_sge_ring) =
4615 			BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4616 					BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4617 		if (!bnx2x_fp(bp, index, rx_sge_ring))
4618 			goto alloc_mem_err;
4619 		/* RX BD ring */
4620 		bnx2x_set_next_page_rx_bd(fp);
4621 
4622 		/* CQ ring */
4623 		bnx2x_set_next_page_rx_cq(fp);
4624 
4625 		/* BDs */
4626 		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4627 		if (ring_size < rx_ring_size)
4628 			goto alloc_mem_err;
4629 	}
4630 
4631 	return 0;
4632 
4633 /* handles low memory cases */
4634 alloc_mem_err:
4635 	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4636 						index, ring_size);
4637 	/* FW will drop all packets if queue is not big enough,
4638 	 * In these cases we disable the queue
4639 	 * Min size is different for OOO, TPA and non-TPA queues
4640 	 */
4641 	if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4642 				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4643 			/* release memory allocated for this queue */
4644 			bnx2x_free_fp_mem_at(bp, index);
4645 			return -ENOMEM;
4646 	}
4647 	return 0;
4648 }
4649 
bnx2x_alloc_fp_mem_cnic(struct bnx2x * bp)4650 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4651 {
4652 	if (!NO_FCOE(bp))
4653 		/* FCoE */
4654 		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4655 			/* we will fail load process instead of mark
4656 			 * NO_FCOE_FLAG
4657 			 */
4658 			return -ENOMEM;
4659 
4660 	return 0;
4661 }
4662 
bnx2x_alloc_fp_mem(struct bnx2x * bp)4663 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4664 {
4665 	int i;
4666 
4667 	/* 1. Allocate FP for leading - fatal if error
4668 	 * 2. Allocate RSS - fix number of queues if error
4669 	 */
4670 
4671 	/* leading */
4672 	if (bnx2x_alloc_fp_mem_at(bp, 0))
4673 		return -ENOMEM;
4674 
4675 	/* RSS */
4676 	for_each_nondefault_eth_queue(bp, i)
4677 		if (bnx2x_alloc_fp_mem_at(bp, i))
4678 			break;
4679 
4680 	/* handle memory failures */
4681 	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4682 		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4683 
4684 		WARN_ON(delta < 0);
4685 		bnx2x_shrink_eth_fp(bp, delta);
4686 		if (CNIC_SUPPORT(bp))
4687 			/* move non eth FPs next to last eth FP
4688 			 * must be done in that order
4689 			 * FCOE_IDX < FWD_IDX < OOO_IDX
4690 			 */
4691 
4692 			/* move FCoE fp even NO_FCOE_FLAG is on */
4693 			bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4694 		bp->num_ethernet_queues -= delta;
4695 		bp->num_queues = bp->num_ethernet_queues +
4696 				 bp->num_cnic_queues;
4697 		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4698 			  bp->num_queues + delta, bp->num_queues);
4699 	}
4700 
4701 	return 0;
4702 }
4703 
bnx2x_free_mem_bp(struct bnx2x * bp)4704 void bnx2x_free_mem_bp(struct bnx2x *bp)
4705 {
4706 	int i;
4707 
4708 	for (i = 0; i < bp->fp_array_size; i++)
4709 		kfree(bp->fp[i].tpa_info);
4710 	kfree(bp->fp);
4711 	kfree(bp->sp_objs);
4712 	kfree(bp->fp_stats);
4713 	kfree(bp->bnx2x_txq);
4714 	kfree(bp->msix_table);
4715 	kfree(bp->ilt);
4716 }
4717 
bnx2x_alloc_mem_bp(struct bnx2x * bp)4718 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4719 {
4720 	struct bnx2x_fastpath *fp;
4721 	struct msix_entry *tbl;
4722 	struct bnx2x_ilt *ilt;
4723 	int msix_table_size = 0;
4724 	int fp_array_size, txq_array_size;
4725 	int i;
4726 
4727 	/*
4728 	 * The biggest MSI-X table we might need is as a maximum number of fast
4729 	 * path IGU SBs plus default SB (for PF only).
4730 	 */
4731 	msix_table_size = bp->igu_sb_cnt;
4732 	if (IS_PF(bp))
4733 		msix_table_size++;
4734 	BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4735 
4736 	/* fp array: RSS plus CNIC related L2 queues */
4737 	fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4738 	bp->fp_array_size = fp_array_size;
4739 	BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4740 
4741 	fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4742 	if (!fp)
4743 		goto alloc_err;
4744 	for (i = 0; i < bp->fp_array_size; i++) {
4745 		fp[i].tpa_info =
4746 			kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4747 				sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4748 		if (!(fp[i].tpa_info))
4749 			goto alloc_err;
4750 	}
4751 
4752 	bp->fp = fp;
4753 
4754 	/* allocate sp objs */
4755 	bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4756 			      GFP_KERNEL);
4757 	if (!bp->sp_objs)
4758 		goto alloc_err;
4759 
4760 	/* allocate fp_stats */
4761 	bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4762 			       GFP_KERNEL);
4763 	if (!bp->fp_stats)
4764 		goto alloc_err;
4765 
4766 	/* Allocate memory for the transmission queues array */
4767 	txq_array_size =
4768 		BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4769 	BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4770 
4771 	bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4772 				GFP_KERNEL);
4773 	if (!bp->bnx2x_txq)
4774 		goto alloc_err;
4775 
4776 	/* msix table */
4777 	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4778 	if (!tbl)
4779 		goto alloc_err;
4780 	bp->msix_table = tbl;
4781 
4782 	/* ilt */
4783 	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4784 	if (!ilt)
4785 		goto alloc_err;
4786 	bp->ilt = ilt;
4787 
4788 	return 0;
4789 alloc_err:
4790 	bnx2x_free_mem_bp(bp);
4791 	return -ENOMEM;
4792 }
4793 
bnx2x_reload_if_running(struct net_device * dev)4794 int bnx2x_reload_if_running(struct net_device *dev)
4795 {
4796 	struct bnx2x *bp = netdev_priv(dev);
4797 
4798 	if (unlikely(!netif_running(dev)))
4799 		return 0;
4800 
4801 	bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4802 	return bnx2x_nic_load(bp, LOAD_NORMAL);
4803 }
4804 
bnx2x_get_cur_phy_idx(struct bnx2x * bp)4805 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4806 {
4807 	u32 sel_phy_idx = 0;
4808 	if (bp->link_params.num_phys <= 1)
4809 		return INT_PHY;
4810 
4811 	if (bp->link_vars.link_up) {
4812 		sel_phy_idx = EXT_PHY1;
4813 		/* In case link is SERDES, check if the EXT_PHY2 is the one */
4814 		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4815 		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4816 			sel_phy_idx = EXT_PHY2;
4817 	} else {
4818 
4819 		switch (bnx2x_phy_selection(&bp->link_params)) {
4820 		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4821 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4822 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4823 		       sel_phy_idx = EXT_PHY1;
4824 		       break;
4825 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4826 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4827 		       sel_phy_idx = EXT_PHY2;
4828 		       break;
4829 		}
4830 	}
4831 
4832 	return sel_phy_idx;
4833 }
bnx2x_get_link_cfg_idx(struct bnx2x * bp)4834 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4835 {
4836 	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4837 	/*
4838 	 * The selected activated PHY is always after swapping (in case PHY
4839 	 * swapping is enabled). So when swapping is enabled, we need to reverse
4840 	 * the configuration
4841 	 */
4842 
4843 	if (bp->link_params.multi_phy_config &
4844 	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4845 		if (sel_phy_idx == EXT_PHY1)
4846 			sel_phy_idx = EXT_PHY2;
4847 		else if (sel_phy_idx == EXT_PHY2)
4848 			sel_phy_idx = EXT_PHY1;
4849 	}
4850 	return LINK_CONFIG_IDX(sel_phy_idx);
4851 }
4852 
4853 #ifdef NETDEV_FCOE_WWNN
bnx2x_fcoe_get_wwn(struct net_device * dev,u64 * wwn,int type)4854 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4855 {
4856 	struct bnx2x *bp = netdev_priv(dev);
4857 	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4858 
4859 	switch (type) {
4860 	case NETDEV_FCOE_WWNN:
4861 		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4862 				cp->fcoe_wwn_node_name_lo);
4863 		break;
4864 	case NETDEV_FCOE_WWPN:
4865 		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4866 				cp->fcoe_wwn_port_name_lo);
4867 		break;
4868 	default:
4869 		BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4870 		return -EINVAL;
4871 	}
4872 
4873 	return 0;
4874 }
4875 #endif
4876 
4877 /* called with rtnl_lock */
bnx2x_change_mtu(struct net_device * dev,int new_mtu)4878 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4879 {
4880 	struct bnx2x *bp = netdev_priv(dev);
4881 
4882 	if (pci_num_vf(bp->pdev)) {
4883 		DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4884 		return -EPERM;
4885 	}
4886 
4887 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4888 		BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4889 		return -EAGAIN;
4890 	}
4891 
4892 	/* This does not race with packet allocation
4893 	 * because the actual alloc size is
4894 	 * only updated as part of load
4895 	 */
4896 	dev->mtu = new_mtu;
4897 
4898 	if (!bnx2x_mtu_allows_gro(new_mtu))
4899 		dev->features &= ~NETIF_F_GRO_HW;
4900 
4901 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4902 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4903 
4904 	return bnx2x_reload_if_running(dev);
4905 }
4906 
bnx2x_fix_features(struct net_device * dev,netdev_features_t features)4907 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4908 				     netdev_features_t features)
4909 {
4910 	struct bnx2x *bp = netdev_priv(dev);
4911 
4912 	if (pci_num_vf(bp->pdev)) {
4913 		netdev_features_t changed = dev->features ^ features;
4914 
4915 		/* Revert the requested changes in features if they
4916 		 * would require internal reload of PF in bnx2x_set_features().
4917 		 */
4918 		if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4919 			features &= ~NETIF_F_RXCSUM;
4920 			features |= dev->features & NETIF_F_RXCSUM;
4921 		}
4922 
4923 		if (changed & NETIF_F_LOOPBACK) {
4924 			features &= ~NETIF_F_LOOPBACK;
4925 			features |= dev->features & NETIF_F_LOOPBACK;
4926 		}
4927 	}
4928 
4929 	/* TPA requires Rx CSUM offloading */
4930 	if (!(features & NETIF_F_RXCSUM))
4931 		features &= ~NETIF_F_LRO;
4932 
4933 	if (!(features & NETIF_F_GRO) || !bnx2x_mtu_allows_gro(dev->mtu))
4934 		features &= ~NETIF_F_GRO_HW;
4935 	if (features & NETIF_F_GRO_HW)
4936 		features &= ~NETIF_F_LRO;
4937 
4938 	return features;
4939 }
4940 
bnx2x_set_features(struct net_device * dev,netdev_features_t features)4941 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4942 {
4943 	struct bnx2x *bp = netdev_priv(dev);
4944 	netdev_features_t changes = features ^ dev->features;
4945 	bool bnx2x_reload = false;
4946 	int rc;
4947 
4948 	/* VFs or non SRIOV PFs should be able to change loopback feature */
4949 	if (!pci_num_vf(bp->pdev)) {
4950 		if (features & NETIF_F_LOOPBACK) {
4951 			if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4952 				bp->link_params.loopback_mode = LOOPBACK_BMAC;
4953 				bnx2x_reload = true;
4954 			}
4955 		} else {
4956 			if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4957 				bp->link_params.loopback_mode = LOOPBACK_NONE;
4958 				bnx2x_reload = true;
4959 			}
4960 		}
4961 	}
4962 
4963 	/* Don't care about GRO changes */
4964 	changes &= ~NETIF_F_GRO;
4965 
4966 	if (changes)
4967 		bnx2x_reload = true;
4968 
4969 	if (bnx2x_reload) {
4970 		if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4971 			dev->features = features;
4972 			rc = bnx2x_reload_if_running(dev);
4973 			return rc ? rc : 1;
4974 		}
4975 		/* else: bnx2x_nic_load() will be called at end of recovery */
4976 	}
4977 
4978 	return 0;
4979 }
4980 
bnx2x_tx_timeout(struct net_device * dev,unsigned int txqueue)4981 void bnx2x_tx_timeout(struct net_device *dev, unsigned int txqueue)
4982 {
4983 	struct bnx2x *bp = netdev_priv(dev);
4984 
4985 	/* We want the information of the dump logged,
4986 	 * but calling bnx2x_panic() would kill all chances of recovery.
4987 	 */
4988 	if (!bp->panic)
4989 #ifndef BNX2X_STOP_ON_ERROR
4990 		bnx2x_panic_dump(bp, false);
4991 #else
4992 		bnx2x_panic();
4993 #endif
4994 
4995 	/* This allows the netif to be shutdown gracefully before resetting */
4996 	bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4997 }
4998 
bnx2x_suspend(struct device * dev_d)4999 static int __maybe_unused bnx2x_suspend(struct device *dev_d)
5000 {
5001 	struct pci_dev *pdev = to_pci_dev(dev_d);
5002 	struct net_device *dev = pci_get_drvdata(pdev);
5003 	struct bnx2x *bp;
5004 
5005 	if (!dev) {
5006 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5007 		return -ENODEV;
5008 	}
5009 	bp = netdev_priv(dev);
5010 
5011 	rtnl_lock();
5012 
5013 	if (!netif_running(dev)) {
5014 		rtnl_unlock();
5015 		return 0;
5016 	}
5017 
5018 	netif_device_detach(dev);
5019 
5020 	bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
5021 
5022 	rtnl_unlock();
5023 
5024 	return 0;
5025 }
5026 
bnx2x_resume(struct device * dev_d)5027 static int __maybe_unused bnx2x_resume(struct device *dev_d)
5028 {
5029 	struct pci_dev *pdev = to_pci_dev(dev_d);
5030 	struct net_device *dev = pci_get_drvdata(pdev);
5031 	struct bnx2x *bp;
5032 	int rc;
5033 
5034 	if (!dev) {
5035 		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5036 		return -ENODEV;
5037 	}
5038 	bp = netdev_priv(dev);
5039 
5040 	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
5041 		BNX2X_ERR("Handling parity error recovery. Try again later\n");
5042 		return -EAGAIN;
5043 	}
5044 
5045 	rtnl_lock();
5046 
5047 	if (!netif_running(dev)) {
5048 		rtnl_unlock();
5049 		return 0;
5050 	}
5051 
5052 	netif_device_attach(dev);
5053 
5054 	rc = bnx2x_nic_load(bp, LOAD_OPEN);
5055 
5056 	rtnl_unlock();
5057 
5058 	return rc;
5059 }
5060 
5061 SIMPLE_DEV_PM_OPS(bnx2x_pm_ops, bnx2x_suspend, bnx2x_resume);
5062 
bnx2x_set_ctx_validation(struct bnx2x * bp,struct eth_context * cxt,u32 cid)5063 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
5064 			      u32 cid)
5065 {
5066 	if (!cxt) {
5067 		BNX2X_ERR("bad context pointer %p\n", cxt);
5068 		return;
5069 	}
5070 
5071 	/* ustorm cxt validation */
5072 	cxt->ustorm_ag_context.cdu_usage =
5073 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5074 			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5075 	/* xcontext validation */
5076 	cxt->xstorm_ag_context.cdu_reserved =
5077 		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5078 			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5079 }
5080 
storm_memset_hc_timeout(struct bnx2x * bp,u8 port,u8 fw_sb_id,u8 sb_index,u8 ticks)5081 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5082 				    u8 fw_sb_id, u8 sb_index,
5083 				    u8 ticks)
5084 {
5085 	u32 addr = BAR_CSTRORM_INTMEM +
5086 		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5087 	REG_WR8(bp, addr, ticks);
5088 	DP(NETIF_MSG_IFUP,
5089 	   "port %x fw_sb_id %d sb_index %d ticks %d\n",
5090 	   port, fw_sb_id, sb_index, ticks);
5091 }
5092 
storm_memset_hc_disable(struct bnx2x * bp,u8 port,u16 fw_sb_id,u8 sb_index,u8 disable)5093 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5094 				    u16 fw_sb_id, u8 sb_index,
5095 				    u8 disable)
5096 {
5097 	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5098 	u32 addr = BAR_CSTRORM_INTMEM +
5099 		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5100 	u8 flags = REG_RD8(bp, addr);
5101 	/* clear and set */
5102 	flags &= ~HC_INDEX_DATA_HC_ENABLED;
5103 	flags |= enable_flag;
5104 	REG_WR8(bp, addr, flags);
5105 	DP(NETIF_MSG_IFUP,
5106 	   "port %x fw_sb_id %d sb_index %d disable %d\n",
5107 	   port, fw_sb_id, sb_index, disable);
5108 }
5109 
bnx2x_update_coalesce_sb_index(struct bnx2x * bp,u8 fw_sb_id,u8 sb_index,u8 disable,u16 usec)5110 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5111 				    u8 sb_index, u8 disable, u16 usec)
5112 {
5113 	int port = BP_PORT(bp);
5114 	u8 ticks = usec / BNX2X_BTR;
5115 
5116 	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5117 
5118 	disable = disable ? 1 : (usec ? 0 : 1);
5119 	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5120 }
5121 
bnx2x_schedule_sp_rtnl(struct bnx2x * bp,enum sp_rtnl_flag flag,u32 verbose)5122 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5123 			    u32 verbose)
5124 {
5125 	smp_mb__before_atomic();
5126 	set_bit(flag, &bp->sp_rtnl_state);
5127 	smp_mb__after_atomic();
5128 	DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5129 	   flag);
5130 	schedule_delayed_work(&bp->sp_rtnl_task, 0);
5131 }
5132