1 /* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #ifndef IRDMA_TYPE_H
4 #define IRDMA_TYPE_H
5 #include "osdep.h"
6 #include "irdma.h"
7 #include "user.h"
8 #include "hmc.h"
9 #include "uda.h"
10 #include "ws.h"
11 #define IRDMA_DEBUG_ERR		"ERR"
12 #define IRDMA_DEBUG_INIT	"INIT"
13 #define IRDMA_DEBUG_DEV		"DEV"
14 #define IRDMA_DEBUG_CM		"CM"
15 #define IRDMA_DEBUG_VERBS	"VERBS"
16 #define IRDMA_DEBUG_PUDA	"PUDA"
17 #define IRDMA_DEBUG_ILQ		"ILQ"
18 #define IRDMA_DEBUG_IEQ		"IEQ"
19 #define IRDMA_DEBUG_QP		"QP"
20 #define IRDMA_DEBUG_CQ		"CQ"
21 #define IRDMA_DEBUG_MR		"MR"
22 #define IRDMA_DEBUG_PBLE	"PBLE"
23 #define IRDMA_DEBUG_WQE		"WQE"
24 #define IRDMA_DEBUG_AEQ		"AEQ"
25 #define IRDMA_DEBUG_CQP		"CQP"
26 #define IRDMA_DEBUG_HMC		"HMC"
27 #define IRDMA_DEBUG_USER	"USER"
28 #define IRDMA_DEBUG_VIRT	"VIRT"
29 #define IRDMA_DEBUG_DCB		"DCB"
30 #define	IRDMA_DEBUG_CQE		"CQE"
31 #define IRDMA_DEBUG_CLNT	"CLNT"
32 #define IRDMA_DEBUG_WS		"WS"
33 #define IRDMA_DEBUG_STATS	"STATS"
34 
35 enum irdma_page_size {
36 	IRDMA_PAGE_SIZE_4K = 0,
37 	IRDMA_PAGE_SIZE_2M,
38 	IRDMA_PAGE_SIZE_1G,
39 };
40 
41 enum irdma_hdrct_flags {
42 	DDP_LEN_FLAG  = 0x80,
43 	DDP_HDR_FLAG  = 0x40,
44 	RDMA_HDR_FLAG = 0x20,
45 };
46 
47 enum irdma_term_layers {
48 	LAYER_RDMA = 0,
49 	LAYER_DDP  = 1,
50 	LAYER_MPA  = 2,
51 };
52 
53 enum irdma_term_error_types {
54 	RDMAP_REMOTE_PROT = 1,
55 	RDMAP_REMOTE_OP   = 2,
56 	DDP_CATASTROPHIC  = 0,
57 	DDP_TAGGED_BUF    = 1,
58 	DDP_UNTAGGED_BUF  = 2,
59 	DDP_LLP		  = 3,
60 };
61 
62 enum irdma_term_rdma_errors {
63 	RDMAP_INV_STAG		  = 0x00,
64 	RDMAP_INV_BOUNDS	  = 0x01,
65 	RDMAP_ACCESS		  = 0x02,
66 	RDMAP_UNASSOC_STAG	  = 0x03,
67 	RDMAP_TO_WRAP		  = 0x04,
68 	RDMAP_INV_RDMAP_VER       = 0x05,
69 	RDMAP_UNEXPECTED_OP       = 0x06,
70 	RDMAP_CATASTROPHIC_LOCAL  = 0x07,
71 	RDMAP_CATASTROPHIC_GLOBAL = 0x08,
72 	RDMAP_CANT_INV_STAG       = 0x09,
73 	RDMAP_UNSPECIFIED	  = 0xff,
74 };
75 
76 enum irdma_term_ddp_errors {
77 	DDP_CATASTROPHIC_LOCAL      = 0x00,
78 	DDP_TAGGED_INV_STAG	    = 0x00,
79 	DDP_TAGGED_BOUNDS	    = 0x01,
80 	DDP_TAGGED_UNASSOC_STAG     = 0x02,
81 	DDP_TAGGED_TO_WRAP	    = 0x03,
82 	DDP_TAGGED_INV_DDP_VER      = 0x04,
83 	DDP_UNTAGGED_INV_QN	    = 0x01,
84 	DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
85 	DDP_UNTAGGED_INV_MSN_RANGE  = 0x03,
86 	DDP_UNTAGGED_INV_MO	    = 0x04,
87 	DDP_UNTAGGED_INV_TOO_LONG   = 0x05,
88 	DDP_UNTAGGED_INV_DDP_VER    = 0x06,
89 };
90 
91 enum irdma_term_mpa_errors {
92 	MPA_CLOSED  = 0x01,
93 	MPA_CRC     = 0x02,
94 	MPA_MARKER  = 0x03,
95 	MPA_REQ_RSP = 0x04,
96 };
97 
98 enum irdma_qp_event_type {
99 	IRDMA_QP_EVENT_CATASTROPHIC,
100 	IRDMA_QP_EVENT_ACCESS_ERR,
101 	IRDMA_QP_EVENT_REQ_ERR,
102 };
103 
104 enum irdma_hw_stats_index_32b {
105 	IRDMA_HW_STAT_INDEX_IP4RXDISCARD	= 0,
106 	IRDMA_HW_STAT_INDEX_IP4RXTRUNC		= 1,
107 	IRDMA_HW_STAT_INDEX_IP4TXNOROUTE	= 2,
108 	IRDMA_HW_STAT_INDEX_IP6RXDISCARD	= 3,
109 	IRDMA_HW_STAT_INDEX_IP6RXTRUNC		= 4,
110 	IRDMA_HW_STAT_INDEX_IP6TXNOROUTE	= 5,
111 	IRDMA_HW_STAT_INDEX_TCPRTXSEG		= 6,
112 	IRDMA_HW_STAT_INDEX_TCPRXOPTERR		= 7,
113 	IRDMA_HW_STAT_INDEX_TCPRXPROTOERR	= 8,
114 	IRDMA_HW_STAT_INDEX_MAX_32_GEN_1	= 9, /* Must be same value as next entry */
115 	IRDMA_HW_STAT_INDEX_RXVLANERR		= 9,
116 	IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED	= 10,
117 	IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED	= 11,
118 	IRDMA_HW_STAT_INDEX_TXNPCNPSENT		= 12,
119 	IRDMA_HW_STAT_INDEX_MAX_32, /* Must be last entry */
120 };
121 
122 enum irdma_hw_stats_index_64b {
123 	IRDMA_HW_STAT_INDEX_IP4RXOCTS	= 0,
124 	IRDMA_HW_STAT_INDEX_IP4RXPKTS	= 1,
125 	IRDMA_HW_STAT_INDEX_IP4RXFRAGS	= 2,
126 	IRDMA_HW_STAT_INDEX_IP4RXMCPKTS	= 3,
127 	IRDMA_HW_STAT_INDEX_IP4TXOCTS	= 4,
128 	IRDMA_HW_STAT_INDEX_IP4TXPKTS	= 5,
129 	IRDMA_HW_STAT_INDEX_IP4TXFRAGS	= 6,
130 	IRDMA_HW_STAT_INDEX_IP4TXMCPKTS	= 7,
131 	IRDMA_HW_STAT_INDEX_IP6RXOCTS	= 8,
132 	IRDMA_HW_STAT_INDEX_IP6RXPKTS	= 9,
133 	IRDMA_HW_STAT_INDEX_IP6RXFRAGS	= 10,
134 	IRDMA_HW_STAT_INDEX_IP6RXMCPKTS	= 11,
135 	IRDMA_HW_STAT_INDEX_IP6TXOCTS	= 12,
136 	IRDMA_HW_STAT_INDEX_IP6TXPKTS	= 13,
137 	IRDMA_HW_STAT_INDEX_IP6TXFRAGS	= 14,
138 	IRDMA_HW_STAT_INDEX_IP6TXMCPKTS	= 15,
139 	IRDMA_HW_STAT_INDEX_TCPRXSEGS	= 16,
140 	IRDMA_HW_STAT_INDEX_TCPTXSEG	= 17,
141 	IRDMA_HW_STAT_INDEX_RDMARXRDS	= 18,
142 	IRDMA_HW_STAT_INDEX_RDMARXSNDS	= 19,
143 	IRDMA_HW_STAT_INDEX_RDMARXWRS	= 20,
144 	IRDMA_HW_STAT_INDEX_RDMATXRDS	= 21,
145 	IRDMA_HW_STAT_INDEX_RDMATXSNDS	= 22,
146 	IRDMA_HW_STAT_INDEX_RDMATXWRS	= 23,
147 	IRDMA_HW_STAT_INDEX_RDMAVBND	= 24,
148 	IRDMA_HW_STAT_INDEX_RDMAVINV	= 25,
149 	IRDMA_HW_STAT_INDEX_MAX_64_GEN_1 = 26, /* Must be same value as next entry */
150 	IRDMA_HW_STAT_INDEX_IP4RXMCOCTS	= 26,
151 	IRDMA_HW_STAT_INDEX_IP4TXMCOCTS	= 27,
152 	IRDMA_HW_STAT_INDEX_IP6RXMCOCTS	= 28,
153 	IRDMA_HW_STAT_INDEX_IP6TXMCOCTS	= 29,
154 	IRDMA_HW_STAT_INDEX_UDPRXPKTS	= 30,
155 	IRDMA_HW_STAT_INDEX_UDPTXPKTS	= 31,
156 	IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 32,
157 	IRDMA_HW_STAT_INDEX_MAX_64, /* Must be last entry */
158 };
159 
160 enum irdma_feature_type {
161 	IRDMA_FEATURE_FW_INFO = 0,
162 	IRDMA_HW_VERSION_INFO = 1,
163 	IRDMA_QSETS_MAX       = 26,
164 	IRDMA_MAX_FEATURES, /* Must be last entry */
165 };
166 
167 enum irdma_sched_prio_type {
168 	IRDMA_PRIO_WEIGHTED_RR     = 1,
169 	IRDMA_PRIO_STRICT	   = 2,
170 	IRDMA_PRIO_WEIGHTED_STRICT = 3,
171 };
172 
173 enum irdma_vm_vf_type {
174 	IRDMA_VF_TYPE = 0,
175 	IRDMA_VM_TYPE,
176 	IRDMA_PF_TYPE,
177 };
178 
179 enum irdma_cqp_hmc_profile {
180 	IRDMA_HMC_PROFILE_DEFAULT  = 1,
181 	IRDMA_HMC_PROFILE_FAVOR_VF = 2,
182 	IRDMA_HMC_PROFILE_EQUAL    = 3,
183 };
184 
185 enum irdma_quad_entry_type {
186 	IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
187 	IRDMA_QHASH_TYPE_TCP_SYN,
188 	IRDMA_QHASH_TYPE_UDP_UNICAST,
189 	IRDMA_QHASH_TYPE_UDP_MCAST,
190 	IRDMA_QHASH_TYPE_ROCE_MCAST,
191 	IRDMA_QHASH_TYPE_ROCEV2_HW,
192 };
193 
194 enum irdma_quad_hash_manage_type {
195 	IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
196 	IRDMA_QHASH_MANAGE_TYPE_ADD,
197 	IRDMA_QHASH_MANAGE_TYPE_MODIFY,
198 };
199 
200 enum irdma_syn_rst_handling {
201 	IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
202 	IRDMA_SYN_RST_HANDLING_HW_TCP,
203 	IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
204 	IRDMA_SYN_RST_HANDLING_FW_TCP,
205 };
206 
207 enum irdma_queue_type {
208 	IRDMA_QUEUE_TYPE_SQ_RQ = 0,
209 	IRDMA_QUEUE_TYPE_CQP,
210 };
211 
212 struct irdma_sc_dev;
213 struct irdma_vsi_pestat;
214 
215 struct irdma_dcqcn_cc_params {
216 	u8 cc_cfg_valid;
217 	u8 min_dec_factor;
218 	u8 min_rate;
219 	u8 dcqcn_f;
220 	u16 rai_factor;
221 	u16 hai_factor;
222 	u16 dcqcn_t;
223 	u32 dcqcn_b;
224 	u32 rreduce_mperiod;
225 };
226 
227 struct irdma_cqp_init_info {
228 	u64 cqp_compl_ctx;
229 	u64 host_ctx_pa;
230 	u64 sq_pa;
231 	struct irdma_sc_dev *dev;
232 	struct irdma_cqp_quanta *sq;
233 	struct irdma_dcqcn_cc_params dcqcn_params;
234 	__le64 *host_ctx;
235 	u64 *scratch_array;
236 	u32 sq_size;
237 	u16 hw_maj_ver;
238 	u16 hw_min_ver;
239 	u8 struct_ver;
240 	u8 hmc_profile;
241 	u8 ena_vf_count;
242 	u8 ceqs_per_vf;
243 	bool en_datacenter_tcp:1;
244 	bool disable_packed:1;
245 	bool rocev2_rto_policy:1;
246 	enum irdma_protocol_used protocol_used;
247 };
248 
249 struct irdma_terminate_hdr {
250 	u8 layer_etype;
251 	u8 error_code;
252 	u8 hdrct;
253 	u8 rsvd;
254 };
255 
256 struct irdma_cqp_sq_wqe {
257 	__le64 buf[IRDMA_CQP_WQE_SIZE];
258 };
259 
260 struct irdma_sc_aeqe {
261 	__le64 buf[IRDMA_AEQE_SIZE];
262 };
263 
264 struct irdma_ceqe {
265 	__le64 buf[IRDMA_CEQE_SIZE];
266 };
267 
268 struct irdma_cqp_ctx {
269 	__le64 buf[IRDMA_CQP_CTX_SIZE];
270 };
271 
272 struct irdma_cq_shadow_area {
273 	__le64 buf[IRDMA_SHADOW_AREA_SIZE];
274 };
275 
276 struct irdma_dev_hw_stats_offsets {
277 	u32 stats_offset_32[IRDMA_HW_STAT_INDEX_MAX_32];
278 	u32 stats_offset_64[IRDMA_HW_STAT_INDEX_MAX_64];
279 };
280 
281 struct irdma_dev_hw_stats {
282 	u64 stats_val_32[IRDMA_HW_STAT_INDEX_MAX_32];
283 	u64 stats_val_64[IRDMA_HW_STAT_INDEX_MAX_64];
284 };
285 
286 struct irdma_gather_stats {
287 	u32 rsvd1;
288 	u32 rxvlanerr;
289 	u64 ip4rxocts;
290 	u64 ip4rxpkts;
291 	u32 ip4rxtrunc;
292 	u32 ip4rxdiscard;
293 	u64 ip4rxfrags;
294 	u64 ip4rxmcocts;
295 	u64 ip4rxmcpkts;
296 	u64 ip6rxocts;
297 	u64 ip6rxpkts;
298 	u32 ip6rxtrunc;
299 	u32 ip6rxdiscard;
300 	u64 ip6rxfrags;
301 	u64 ip6rxmcocts;
302 	u64 ip6rxmcpkts;
303 	u64 ip4txocts;
304 	u64 ip4txpkts;
305 	u64 ip4txfrag;
306 	u64 ip4txmcocts;
307 	u64 ip4txmcpkts;
308 	u64 ip6txocts;
309 	u64 ip6txpkts;
310 	u64 ip6txfrags;
311 	u64 ip6txmcocts;
312 	u64 ip6txmcpkts;
313 	u32 ip6txnoroute;
314 	u32 ip4txnoroute;
315 	u64 tcprxsegs;
316 	u32 tcprxprotoerr;
317 	u32 tcprxopterr;
318 	u64 tcptxsegs;
319 	u32 rsvd2;
320 	u32 tcprtxseg;
321 	u64 udprxpkts;
322 	u64 udptxpkts;
323 	u64 rdmarxwrs;
324 	u64 rdmarxrds;
325 	u64 rdmarxsnds;
326 	u64 rdmatxwrs;
327 	u64 rdmatxrds;
328 	u64 rdmatxsnds;
329 	u64 rdmavbn;
330 	u64 rdmavinv;
331 	u64 rxnpecnmrkpkts;
332 	u32 rxrpcnphandled;
333 	u32 rxrpcnpignored;
334 	u32 txnpcnpsent;
335 	u32 rsvd3[88];
336 };
337 
338 struct irdma_stats_gather_info {
339 	bool use_hmc_fcn_index:1;
340 	bool use_stats_inst:1;
341 	u8 hmc_fcn_index;
342 	u8 stats_inst_index;
343 	struct irdma_dma_mem stats_buff_mem;
344 	void *gather_stats_va;
345 	void *last_gather_stats_va;
346 };
347 
348 struct irdma_vsi_pestat {
349 	struct irdma_hw *hw;
350 	struct irdma_dev_hw_stats hw_stats;
351 	struct irdma_stats_gather_info gather_info;
352 	struct timer_list stats_timer;
353 	struct irdma_sc_vsi *vsi;
354 	struct irdma_dev_hw_stats last_hw_stats;
355 	spinlock_t lock; /* rdma stats lock */
356 };
357 
358 struct irdma_hw {
359 	u8 __iomem *hw_addr;
360 	u8 __iomem *priv_hw_addr;
361 	struct device *device;
362 	struct irdma_hmc_info hmc;
363 };
364 
365 struct irdma_pfpdu {
366 	struct list_head rxlist;
367 	u32 rcv_nxt;
368 	u32 fps;
369 	u32 max_fpdu_data;
370 	u32 nextseqnum;
371 	u32 rcv_start_seq;
372 	bool mode:1;
373 	bool mpa_crc_err:1;
374 	u8  marker_len;
375 	u64 total_ieq_bufs;
376 	u64 fpdu_processed;
377 	u64 bad_seq_num;
378 	u64 crc_err;
379 	u64 no_tx_bufs;
380 	u64 tx_err;
381 	u64 out_of_order;
382 	u64 pmode_count;
383 	struct irdma_sc_ah *ah;
384 	struct irdma_puda_buf *ah_buf;
385 	spinlock_t lock; /* fpdu processing lock */
386 	struct irdma_puda_buf *lastrcv_buf;
387 };
388 
389 struct irdma_sc_pd {
390 	struct irdma_sc_dev *dev;
391 	u32 pd_id;
392 	int abi_ver;
393 };
394 
395 struct irdma_cqp_quanta {
396 	__le64 elem[IRDMA_CQP_WQE_SIZE];
397 };
398 
399 struct irdma_sc_cqp {
400 	u32 size;
401 	u64 sq_pa;
402 	u64 host_ctx_pa;
403 	void *back_cqp;
404 	struct irdma_sc_dev *dev;
405 	int (*process_cqp_sds)(struct irdma_sc_dev *dev,
406 			       struct irdma_update_sds_info *info);
407 	struct irdma_dma_mem sdbuf;
408 	struct irdma_ring sq_ring;
409 	struct irdma_cqp_quanta *sq_base;
410 	struct irdma_dcqcn_cc_params dcqcn_params;
411 	__le64 *host_ctx;
412 	u64 *scratch_array;
413 	u32 cqp_id;
414 	u32 sq_size;
415 	u32 hw_sq_size;
416 	u16 hw_maj_ver;
417 	u16 hw_min_ver;
418 	u8 struct_ver;
419 	u8 polarity;
420 	u8 hmc_profile;
421 	u8 ena_vf_count;
422 	u8 timeout_count;
423 	u8 ceqs_per_vf;
424 	bool en_datacenter_tcp:1;
425 	bool disable_packed:1;
426 	bool rocev2_rto_policy:1;
427 	enum irdma_protocol_used protocol_used;
428 };
429 
430 struct irdma_sc_aeq {
431 	u32 size;
432 	u64 aeq_elem_pa;
433 	struct irdma_sc_dev *dev;
434 	struct irdma_sc_aeqe *aeqe_base;
435 	void *pbl_list;
436 	u32 elem_cnt;
437 	struct irdma_ring aeq_ring;
438 	u8 pbl_chunk_size;
439 	u32 first_pm_pbl_idx;
440 	u32 msix_idx;
441 	u8 polarity;
442 	bool virtual_map:1;
443 };
444 
445 struct irdma_sc_ceq {
446 	u32 size;
447 	u64 ceq_elem_pa;
448 	struct irdma_sc_dev *dev;
449 	struct irdma_ceqe *ceqe_base;
450 	void *pbl_list;
451 	u32 ceq_id;
452 	u32 elem_cnt;
453 	struct irdma_ring ceq_ring;
454 	u8 pbl_chunk_size;
455 	u8 tph_val;
456 	u32 first_pm_pbl_idx;
457 	u8 polarity;
458 	struct irdma_sc_vsi *vsi;
459 	struct irdma_sc_cq **reg_cq;
460 	u32 reg_cq_size;
461 	spinlock_t req_cq_lock; /* protect access to reg_cq array */
462 	bool virtual_map:1;
463 	bool tph_en:1;
464 	bool itr_no_expire:1;
465 };
466 
467 struct irdma_sc_cq {
468 	struct irdma_cq_uk cq_uk;
469 	u64 cq_pa;
470 	u64 shadow_area_pa;
471 	struct irdma_sc_dev *dev;
472 	struct irdma_sc_vsi *vsi;
473 	void *pbl_list;
474 	void *back_cq;
475 	u32 ceq_id;
476 	u32 shadow_read_threshold;
477 	u8 pbl_chunk_size;
478 	u8 cq_type;
479 	u8 tph_val;
480 	u32 first_pm_pbl_idx;
481 	bool ceqe_mask:1;
482 	bool virtual_map:1;
483 	bool check_overflow:1;
484 	bool ceq_id_valid:1;
485 	bool tph_en;
486 };
487 
488 struct irdma_sc_qp {
489 	struct irdma_qp_uk qp_uk;
490 	u64 sq_pa;
491 	u64 rq_pa;
492 	u64 hw_host_ctx_pa;
493 	u64 shadow_area_pa;
494 	u64 q2_pa;
495 	struct irdma_sc_dev *dev;
496 	struct irdma_sc_vsi *vsi;
497 	struct irdma_sc_pd *pd;
498 	__le64 *hw_host_ctx;
499 	void *llp_stream_handle;
500 	struct irdma_pfpdu pfpdu;
501 	u32 ieq_qp;
502 	u8 *q2_buf;
503 	u64 qp_compl_ctx;
504 	u32 push_idx;
505 	u16 qs_handle;
506 	u16 push_offset;
507 	u8 flush_wqes_count;
508 	u8 sq_tph_val;
509 	u8 rq_tph_val;
510 	u8 qp_state;
511 	u8 hw_sq_size;
512 	u8 hw_rq_size;
513 	u8 src_mac_addr_idx;
514 	bool on_qoslist:1;
515 	bool ieq_pass_thru:1;
516 	bool sq_tph_en:1;
517 	bool rq_tph_en:1;
518 	bool rcv_tph_en:1;
519 	bool xmit_tph_en:1;
520 	bool virtual_map:1;
521 	bool flush_sq:1;
522 	bool flush_rq:1;
523 	bool sq_flush_code:1;
524 	bool rq_flush_code:1;
525 	enum irdma_flush_opcode flush_code;
526 	enum irdma_qp_event_type event_type;
527 	u8 term_flags;
528 	u8 user_pri;
529 	struct list_head list;
530 };
531 
532 struct irdma_stats_inst_info {
533 	bool use_hmc_fcn_index;
534 	u8 hmc_fn_id;
535 	u8 stats_idx;
536 };
537 
538 struct irdma_up_info {
539 	u8 map[8];
540 	u8 cnp_up_override;
541 	u8 hmc_fcn_idx;
542 	bool use_vlan:1;
543 	bool use_cnp_up_override:1;
544 };
545 
546 #define IRDMA_MAX_WS_NODES	0x3FF
547 #define IRDMA_WS_NODE_INVALID	0xFFFF
548 
549 struct irdma_ws_node_info {
550 	u16 id;
551 	u16 vsi;
552 	u16 parent_id;
553 	u16 qs_handle;
554 	bool type_leaf:1;
555 	bool enable:1;
556 	u8 prio_type;
557 	u8 tc;
558 	u8 weight;
559 };
560 
561 struct irdma_hmc_fpm_misc {
562 	u32 max_ceqs;
563 	u32 max_sds;
564 	u32 xf_block_size;
565 	u32 q1_block_size;
566 	u32 ht_multiplier;
567 	u32 timer_bucket;
568 	u32 rrf_block_size;
569 	u32 ooiscf_block_size;
570 };
571 
572 #define IRDMA_LEAF_DEFAULT_REL_BW		64
573 #define IRDMA_PARENT_DEFAULT_REL_BW		1
574 
575 struct irdma_qos {
576 	struct list_head qplist;
577 	struct mutex qos_mutex; /* protect QoS attributes per QoS level */
578 	u64 lan_qos_handle;
579 	u32 l2_sched_node_id;
580 	u16 qs_handle;
581 	u8 traffic_class;
582 	u8 rel_bw;
583 	u8 prio_type;
584 	bool valid;
585 };
586 
587 #define IRDMA_INVALID_FCN_ID 0xff
588 struct irdma_sc_vsi {
589 	u16 vsi_idx;
590 	struct irdma_sc_dev *dev;
591 	void *back_vsi;
592 	u32 ilq_count;
593 	struct irdma_virt_mem ilq_mem;
594 	struct irdma_puda_rsrc *ilq;
595 	u32 ieq_count;
596 	struct irdma_virt_mem ieq_mem;
597 	struct irdma_puda_rsrc *ieq;
598 	u32 exception_lan_q;
599 	u16 mtu;
600 	u16 vm_id;
601 	u8 fcn_id;
602 	enum irdma_vm_vf_type vm_vf_type;
603 	bool stats_fcn_id_alloc:1;
604 	bool tc_change_pending:1;
605 	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
606 	struct irdma_vsi_pestat *pestat;
607 	atomic_t qp_suspend_reqs;
608 	int (*register_qset)(struct irdma_sc_vsi *vsi,
609 			     struct irdma_ws_node *tc_node);
610 	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
611 				struct irdma_ws_node *tc_node);
612 	u8 qos_rel_bw;
613 	u8 qos_prio_type;
614 	u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
615 	bool dscp_mode:1;
616 };
617 
618 struct irdma_sc_dev {
619 	struct list_head cqp_cmd_head; /* head of the CQP command list */
620 	spinlock_t cqp_lock; /* protect CQP list access */
621 	bool fcn_id_array[IRDMA_MAX_STATS_COUNT];
622 	struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
623 	u64 fpm_query_buf_pa;
624 	u64 fpm_commit_buf_pa;
625 	__le64 *fpm_query_buf;
626 	__le64 *fpm_commit_buf;
627 	struct irdma_hw *hw;
628 	u8 __iomem *db_addr;
629 	u32 __iomem *wqe_alloc_db;
630 	u32 __iomem *cq_arm_db;
631 	u32 __iomem *aeq_alloc_db;
632 	u32 __iomem *cqp_db;
633 	u32 __iomem *cq_ack_db;
634 	u32 __iomem *ceq_itr_mask_db;
635 	u32 __iomem *aeq_itr_mask_db;
636 	u32 __iomem *hw_regs[IRDMA_MAX_REGS];
637 	u32 ceq_itr;   /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
638 	u64 hw_masks[IRDMA_MAX_MASKS];
639 	u64 hw_shifts[IRDMA_MAX_SHIFTS];
640 	u64 hw_stats_regs_32[IRDMA_HW_STAT_INDEX_MAX_32];
641 	u64 hw_stats_regs_64[IRDMA_HW_STAT_INDEX_MAX_64];
642 	u64 feature_info[IRDMA_MAX_FEATURES];
643 	u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
644 	struct irdma_hw_attrs hw_attrs;
645 	struct irdma_hmc_info *hmc_info;
646 	struct irdma_sc_cqp *cqp;
647 	struct irdma_sc_aeq *aeq;
648 	struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
649 	struct irdma_sc_cq *ccq;
650 	const struct irdma_irq_ops *irq_ops;
651 	struct irdma_hmc_fpm_misc hmc_fpm_misc;
652 	struct irdma_ws_node *ws_tree_root;
653 	struct mutex ws_mutex; /* ws tree mutex */
654 	u16 num_vfs;
655 	u8 hmc_fn_id;
656 	u8 vf_id;
657 	bool vchnl_up:1;
658 	bool ceq_valid:1;
659 	u8 pci_rev;
660 	int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
661 	void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
662 	void (*ws_reset)(struct irdma_sc_vsi *vsi);
663 };
664 
665 struct irdma_modify_cq_info {
666 	u64 cq_pa;
667 	struct irdma_cqe *cq_base;
668 	u32 cq_size;
669 	u32 shadow_read_threshold;
670 	u8 pbl_chunk_size;
671 	u32 first_pm_pbl_idx;
672 	bool virtual_map:1;
673 	bool check_overflow;
674 	bool cq_resize:1;
675 };
676 
677 struct irdma_create_qp_info {
678 	bool ord_valid:1;
679 	bool tcp_ctx_valid:1;
680 	bool cq_num_valid:1;
681 	bool arp_cache_idx_valid:1;
682 	bool mac_valid:1;
683 	bool force_lpb;
684 	u8 next_iwarp_state;
685 };
686 
687 struct irdma_modify_qp_info {
688 	u64 rx_win0;
689 	u64 rx_win1;
690 	u16 new_mss;
691 	u8 next_iwarp_state;
692 	u8 curr_iwarp_state;
693 	u8 termlen;
694 	bool ord_valid:1;
695 	bool tcp_ctx_valid:1;
696 	bool udp_ctx_valid:1;
697 	bool cq_num_valid:1;
698 	bool arp_cache_idx_valid:1;
699 	bool reset_tcp_conn:1;
700 	bool remove_hash_idx:1;
701 	bool dont_send_term:1;
702 	bool dont_send_fin:1;
703 	bool cached_var_valid:1;
704 	bool mss_change:1;
705 	bool force_lpb:1;
706 	bool mac_valid:1;
707 };
708 
709 struct irdma_ccq_cqe_info {
710 	struct irdma_sc_cqp *cqp;
711 	u64 scratch;
712 	u32 op_ret_val;
713 	u16 maj_err_code;
714 	u16 min_err_code;
715 	u8 op_code;
716 	bool error;
717 };
718 
719 struct irdma_dcb_app_info {
720 	u8 priority;
721 	u8 selector;
722 	u16 prot_id;
723 };
724 
725 struct irdma_qos_tc_info {
726 	u64 tc_ctx;
727 	u8 rel_bw;
728 	u8 prio_type;
729 	u8 egress_virt_up;
730 	u8 ingress_virt_up;
731 };
732 
733 struct irdma_l2params {
734 	struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
735 	struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
736 	u32 num_apps;
737 	u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
738 	u16 mtu;
739 	u8 up2tc[IRDMA_MAX_USER_PRIORITY];
740 	u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
741 	u8 num_tc;
742 	u8 vsi_rel_bw;
743 	u8 vsi_prio_type;
744 	bool mtu_changed:1;
745 	bool tc_changed:1;
746 	bool dscp_mode:1;
747 };
748 
749 struct irdma_vsi_init_info {
750 	struct irdma_sc_dev *dev;
751 	void *back_vsi;
752 	struct irdma_l2params *params;
753 	u16 exception_lan_q;
754 	u16 pf_data_vsi_num;
755 	enum irdma_vm_vf_type vm_vf_type;
756 	u16 vm_id;
757 	int (*register_qset)(struct irdma_sc_vsi *vsi,
758 			     struct irdma_ws_node *tc_node);
759 	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
760 				struct irdma_ws_node *tc_node);
761 };
762 
763 struct irdma_vsi_stats_info {
764 	struct irdma_vsi_pestat *pestat;
765 	u8 fcn_id;
766 	bool alloc_fcn_id;
767 };
768 
769 struct irdma_device_init_info {
770 	u64 fpm_query_buf_pa;
771 	u64 fpm_commit_buf_pa;
772 	__le64 *fpm_query_buf;
773 	__le64 *fpm_commit_buf;
774 	struct irdma_hw *hw;
775 	void __iomem *bar0;
776 	u8 hmc_fn_id;
777 };
778 
779 struct irdma_ceq_init_info {
780 	u64 ceqe_pa;
781 	struct irdma_sc_dev *dev;
782 	u64 *ceqe_base;
783 	void *pbl_list;
784 	u32 elem_cnt;
785 	u32 ceq_id;
786 	bool virtual_map:1;
787 	bool tph_en:1;
788 	bool itr_no_expire:1;
789 	u8 pbl_chunk_size;
790 	u8 tph_val;
791 	u32 first_pm_pbl_idx;
792 	struct irdma_sc_vsi *vsi;
793 	struct irdma_sc_cq **reg_cq;
794 	u32 reg_cq_idx;
795 };
796 
797 struct irdma_aeq_init_info {
798 	u64 aeq_elem_pa;
799 	struct irdma_sc_dev *dev;
800 	u32 *aeqe_base;
801 	void *pbl_list;
802 	u32 elem_cnt;
803 	bool virtual_map;
804 	u8 pbl_chunk_size;
805 	u32 first_pm_pbl_idx;
806 	u32 msix_idx;
807 };
808 
809 struct irdma_ccq_init_info {
810 	u64 cq_pa;
811 	u64 shadow_area_pa;
812 	struct irdma_sc_dev *dev;
813 	struct irdma_cqe *cq_base;
814 	__le64 *shadow_area;
815 	void *pbl_list;
816 	u32 num_elem;
817 	u32 ceq_id;
818 	u32 shadow_read_threshold;
819 	bool ceqe_mask:1;
820 	bool ceq_id_valid:1;
821 	bool avoid_mem_cflct:1;
822 	bool virtual_map:1;
823 	bool tph_en:1;
824 	u8 tph_val;
825 	u8 pbl_chunk_size;
826 	u32 first_pm_pbl_idx;
827 	struct irdma_sc_vsi *vsi;
828 };
829 
830 struct irdma_udp_offload_info {
831 	bool ipv4:1;
832 	bool insert_vlan_tag:1;
833 	u8 ttl;
834 	u8 tos;
835 	u16 src_port;
836 	u16 dst_port;
837 	u32 dest_ip_addr[4];
838 	u32 snd_mss;
839 	u16 vlan_tag;
840 	u16 arp_idx;
841 	u32 flow_label;
842 	u8 udp_state;
843 	u32 psn_nxt;
844 	u32 lsn;
845 	u32 epsn;
846 	u32 psn_max;
847 	u32 psn_una;
848 	u32 local_ipaddr[4];
849 	u32 cwnd;
850 	u8 rexmit_thresh;
851 	u8 rnr_nak_thresh;
852 };
853 
854 struct irdma_roce_offload_info {
855 	u16 p_key;
856 	u16 err_rq_idx;
857 	u32 qkey;
858 	u32 dest_qp;
859 	u8 roce_tver;
860 	u8 ack_credits;
861 	u8 err_rq_idx_valid;
862 	u32 pd_id;
863 	u16 ord_size;
864 	u16 ird_size;
865 	bool is_qp1:1;
866 	bool udprivcq_en:1;
867 	bool dcqcn_en:1;
868 	bool rcv_no_icrc:1;
869 	bool wr_rdresp_en:1;
870 	bool bind_en:1;
871 	bool fast_reg_en:1;
872 	bool priv_mode_en:1;
873 	bool rd_en:1;
874 	bool timely_en:1;
875 	bool dctcp_en:1;
876 	bool fw_cc_enable:1;
877 	bool use_stats_inst:1;
878 	u16 t_high;
879 	u16 t_low;
880 	u8 last_byte_sent;
881 	u8 mac_addr[ETH_ALEN];
882 	u8 rtomin;
883 };
884 
885 struct irdma_iwarp_offload_info {
886 	u16 rcv_mark_offset;
887 	u16 snd_mark_offset;
888 	u8 ddp_ver;
889 	u8 rdmap_ver;
890 	u8 iwarp_mode;
891 	u16 err_rq_idx;
892 	u32 pd_id;
893 	u16 ord_size;
894 	u16 ird_size;
895 	bool ib_rd_en:1;
896 	bool align_hdrs:1;
897 	bool rcv_no_mpa_crc:1;
898 	bool err_rq_idx_valid:1;
899 	bool snd_mark_en:1;
900 	bool rcv_mark_en:1;
901 	bool wr_rdresp_en:1;
902 	bool bind_en:1;
903 	bool fast_reg_en:1;
904 	bool priv_mode_en:1;
905 	bool rd_en:1;
906 	bool timely_en:1;
907 	bool use_stats_inst:1;
908 	bool ecn_en:1;
909 	bool dctcp_en:1;
910 	u16 t_high;
911 	u16 t_low;
912 	u8 last_byte_sent;
913 	u8 mac_addr[ETH_ALEN];
914 	u8 rtomin;
915 };
916 
917 struct irdma_tcp_offload_info {
918 	bool ipv4:1;
919 	bool no_nagle:1;
920 	bool insert_vlan_tag:1;
921 	bool time_stamp:1;
922 	bool drop_ooo_seg:1;
923 	bool avoid_stretch_ack:1;
924 	bool wscale:1;
925 	bool ignore_tcp_opt:1;
926 	bool ignore_tcp_uns_opt:1;
927 	u8 cwnd_inc_limit;
928 	u8 dup_ack_thresh;
929 	u8 ttl;
930 	u8 src_mac_addr_idx;
931 	u8 tos;
932 	u16 src_port;
933 	u16 dst_port;
934 	u32 dest_ip_addr[4];
935 	//u32 dest_ip_addr0;
936 	//u32 dest_ip_addr1;
937 	//u32 dest_ip_addr2;
938 	//u32 dest_ip_addr3;
939 	u32 snd_mss;
940 	u16 syn_rst_handling;
941 	u16 vlan_tag;
942 	u16 arp_idx;
943 	u32 flow_label;
944 	u8 tcp_state;
945 	u8 snd_wscale;
946 	u8 rcv_wscale;
947 	u32 time_stamp_recent;
948 	u32 time_stamp_age;
949 	u32 snd_nxt;
950 	u32 snd_wnd;
951 	u32 rcv_nxt;
952 	u32 rcv_wnd;
953 	u32 snd_max;
954 	u32 snd_una;
955 	u32 srtt;
956 	u32 rtt_var;
957 	u32 ss_thresh;
958 	u32 cwnd;
959 	u32 snd_wl1;
960 	u32 snd_wl2;
961 	u32 max_snd_window;
962 	u8 rexmit_thresh;
963 	u32 local_ipaddr[4];
964 };
965 
966 struct irdma_qp_host_ctx_info {
967 	u64 qp_compl_ctx;
968 	union {
969 		struct irdma_tcp_offload_info *tcp_info;
970 		struct irdma_udp_offload_info *udp_info;
971 	};
972 	union {
973 		struct irdma_iwarp_offload_info *iwarp_info;
974 		struct irdma_roce_offload_info *roce_info;
975 	};
976 	u32 send_cq_num;
977 	u32 rcv_cq_num;
978 	u32 rem_endpoint_idx;
979 	u8 stats_idx;
980 	bool srq_valid:1;
981 	bool tcp_info_valid:1;
982 	bool iwarp_info_valid:1;
983 	bool stats_idx_valid:1;
984 	u8 user_pri;
985 };
986 
987 struct irdma_aeqe_info {
988 	u64 compl_ctx;
989 	u32 qp_cq_id;
990 	u16 ae_id;
991 	u16 wqe_idx;
992 	u8 tcp_state;
993 	u8 iwarp_state;
994 	bool qp:1;
995 	bool cq:1;
996 	bool sq:1;
997 	bool rq:1;
998 	bool in_rdrsp_wr:1;
999 	bool out_rdrsp:1;
1000 	bool aeqe_overflow:1;
1001 	u8 q2_data_written;
1002 	u8 ae_src;
1003 };
1004 
1005 struct irdma_allocate_stag_info {
1006 	u64 total_len;
1007 	u64 first_pm_pbl_idx;
1008 	u32 chunk_size;
1009 	u32 stag_idx;
1010 	u32 page_size;
1011 	u32 pd_id;
1012 	u16 access_rights;
1013 	bool remote_access:1;
1014 	bool use_hmc_fcn_index:1;
1015 	bool use_pf_rid:1;
1016 	u8 hmc_fcn_index;
1017 };
1018 
1019 struct irdma_mw_alloc_info {
1020 	u32 mw_stag_index;
1021 	u32 page_size;
1022 	u32 pd_id;
1023 	bool remote_access:1;
1024 	bool mw_wide:1;
1025 	bool mw1_bind_dont_vldt_key:1;
1026 };
1027 
1028 struct irdma_reg_ns_stag_info {
1029 	u64 reg_addr_pa;
1030 	u64 va;
1031 	u64 total_len;
1032 	u32 page_size;
1033 	u32 chunk_size;
1034 	u32 first_pm_pbl_index;
1035 	enum irdma_addressing_type addr_type;
1036 	irdma_stag_index stag_idx;
1037 	u16 access_rights;
1038 	u32 pd_id;
1039 	irdma_stag_key stag_key;
1040 	bool use_hmc_fcn_index:1;
1041 	u8 hmc_fcn_index;
1042 	bool use_pf_rid:1;
1043 };
1044 
1045 struct irdma_fast_reg_stag_info {
1046 	u64 wr_id;
1047 	u64 reg_addr_pa;
1048 	u64 fbo;
1049 	void *va;
1050 	u64 total_len;
1051 	u32 page_size;
1052 	u32 chunk_size;
1053 	u32 first_pm_pbl_index;
1054 	enum irdma_addressing_type addr_type;
1055 	irdma_stag_index stag_idx;
1056 	u16 access_rights;
1057 	u32 pd_id;
1058 	irdma_stag_key stag_key;
1059 	bool local_fence:1;
1060 	bool read_fence:1;
1061 	bool signaled:1;
1062 	bool push_wqe:1;
1063 	bool use_hmc_fcn_index:1;
1064 	u8 hmc_fcn_index;
1065 	bool use_pf_rid:1;
1066 	bool defer_flag:1;
1067 };
1068 
1069 struct irdma_dealloc_stag_info {
1070 	u32 stag_idx;
1071 	u32 pd_id;
1072 	bool mr:1;
1073 	bool dealloc_pbl:1;
1074 };
1075 
1076 struct irdma_register_shared_stag {
1077 	u64 va;
1078 	enum irdma_addressing_type addr_type;
1079 	irdma_stag_index new_stag_idx;
1080 	irdma_stag_index parent_stag_idx;
1081 	u32 access_rights;
1082 	u32 pd_id;
1083 	u32 page_size;
1084 	irdma_stag_key new_stag_key;
1085 };
1086 
1087 struct irdma_qp_init_info {
1088 	struct irdma_qp_uk_init_info qp_uk_init_info;
1089 	struct irdma_sc_pd *pd;
1090 	struct irdma_sc_vsi *vsi;
1091 	__le64 *host_ctx;
1092 	u8 *q2;
1093 	u64 sq_pa;
1094 	u64 rq_pa;
1095 	u64 host_ctx_pa;
1096 	u64 q2_pa;
1097 	u64 shadow_area_pa;
1098 	u8 sq_tph_val;
1099 	u8 rq_tph_val;
1100 	bool sq_tph_en:1;
1101 	bool rq_tph_en:1;
1102 	bool rcv_tph_en:1;
1103 	bool xmit_tph_en:1;
1104 	bool virtual_map:1;
1105 };
1106 
1107 struct irdma_cq_init_info {
1108 	struct irdma_sc_dev *dev;
1109 	u64 cq_base_pa;
1110 	u64 shadow_area_pa;
1111 	u32 ceq_id;
1112 	u32 shadow_read_threshold;
1113 	u8 pbl_chunk_size;
1114 	u32 first_pm_pbl_idx;
1115 	bool virtual_map:1;
1116 	bool ceqe_mask:1;
1117 	bool ceq_id_valid:1;
1118 	bool tph_en:1;
1119 	u8 tph_val;
1120 	u8 type;
1121 	struct irdma_cq_uk_init_info cq_uk_init_info;
1122 	struct irdma_sc_vsi *vsi;
1123 };
1124 
1125 struct irdma_upload_context_info {
1126 	u64 buf_pa;
1127 	u32 qp_id;
1128 	u8 qp_type;
1129 	bool freeze_qp:1;
1130 	bool raw_format:1;
1131 };
1132 
1133 struct irdma_local_mac_entry_info {
1134 	u8 mac_addr[6];
1135 	u16 entry_idx;
1136 };
1137 
1138 struct irdma_add_arp_cache_entry_info {
1139 	u8 mac_addr[ETH_ALEN];
1140 	u32 reach_max;
1141 	u16 arp_index;
1142 	bool permanent;
1143 };
1144 
1145 struct irdma_apbvt_info {
1146 	u16 port;
1147 	bool add;
1148 };
1149 
1150 struct irdma_qhash_table_info {
1151 	struct irdma_sc_vsi *vsi;
1152 	enum irdma_quad_hash_manage_type manage;
1153 	enum irdma_quad_entry_type entry_type;
1154 	bool vlan_valid:1;
1155 	bool ipv4_valid:1;
1156 	u8 mac_addr[ETH_ALEN];
1157 	u16 vlan_id;
1158 	u8 user_pri;
1159 	u32 qp_num;
1160 	u32 dest_ip[4];
1161 	u32 src_ip[4];
1162 	u16 dest_port;
1163 	u16 src_port;
1164 };
1165 
1166 struct irdma_cqp_manage_push_page_info {
1167 	u32 push_idx;
1168 	u16 qs_handle;
1169 	u8 free_page;
1170 	u8 push_page_type;
1171 };
1172 
1173 struct irdma_qp_flush_info {
1174 	u16 sq_minor_code;
1175 	u16 sq_major_code;
1176 	u16 rq_minor_code;
1177 	u16 rq_major_code;
1178 	u16 ae_code;
1179 	u8 ae_src;
1180 	bool sq:1;
1181 	bool rq:1;
1182 	bool userflushcode:1;
1183 	bool generate_ae:1;
1184 };
1185 
1186 struct irdma_gen_ae_info {
1187 	u16 ae_code;
1188 	u8 ae_src;
1189 };
1190 
1191 struct irdma_cqp_timeout {
1192 	u64 compl_cqp_cmds;
1193 	u32 count;
1194 };
1195 
1196 struct irdma_irq_ops {
1197 	void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1198 	void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1199 			      bool enable);
1200 	void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1201 	void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1202 };
1203 
1204 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1205 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1206 			bool check_overflow, bool post_sq);
1207 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1208 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1209 			      struct irdma_ccq_cqe_info *info);
1210 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1211 		      struct irdma_ccq_init_info *info);
1212 
1213 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1214 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1215 
1216 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1217 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1218 		      struct irdma_ceq_init_info *info);
1219 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1220 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1221 
1222 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1223 		      struct irdma_aeq_init_info *info);
1224 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1225 			   struct irdma_aeqe_info *info);
1226 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1227 
1228 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1229 		      int abi_ver);
1230 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1231 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1232 			      struct irdma_sc_dev *dev);
1233 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1234 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1235 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1236 		      struct irdma_cqp_init_info *info);
1237 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1238 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1239 				  struct irdma_ccq_cqe_info *cmpl_info);
1240 int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1241 			   struct irdma_fast_reg_stag_info *info, bool post_sq);
1242 int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1243 		       struct irdma_create_qp_info *info, u64 scratch,
1244 		       bool post_sq);
1245 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1246 			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1247 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1248 			   struct irdma_qp_flush_info *info, u64 scratch,
1249 			   bool post_sq);
1250 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1251 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1252 		       struct irdma_modify_qp_info *info, u64 scratch,
1253 		       bool post_sq);
1254 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1255 			irdma_stag stag);
1256 
1257 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1258 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1259 			struct irdma_qp_host_ctx_info *info);
1260 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1261 			     struct irdma_qp_host_ctx_info *info);
1262 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1263 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1264 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1265 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1266 					u8 hmc_fn_id, bool post_sq,
1267 					bool poll_registers);
1268 
1269 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1270 struct cqp_info {
1271 	union {
1272 		struct {
1273 			struct irdma_sc_qp *qp;
1274 			struct irdma_create_qp_info info;
1275 			u64 scratch;
1276 		} qp_create;
1277 
1278 		struct {
1279 			struct irdma_sc_qp *qp;
1280 			struct irdma_modify_qp_info info;
1281 			u64 scratch;
1282 		} qp_modify;
1283 
1284 		struct {
1285 			struct irdma_sc_qp *qp;
1286 			u64 scratch;
1287 			bool remove_hash_idx;
1288 			bool ignore_mw_bnd;
1289 		} qp_destroy;
1290 
1291 		struct {
1292 			struct irdma_sc_cq *cq;
1293 			u64 scratch;
1294 			bool check_overflow;
1295 		} cq_create;
1296 
1297 		struct {
1298 			struct irdma_sc_cq *cq;
1299 			struct irdma_modify_cq_info info;
1300 			u64 scratch;
1301 		} cq_modify;
1302 
1303 		struct {
1304 			struct irdma_sc_cq *cq;
1305 			u64 scratch;
1306 		} cq_destroy;
1307 
1308 		struct {
1309 			struct irdma_sc_dev *dev;
1310 			struct irdma_allocate_stag_info info;
1311 			u64 scratch;
1312 		} alloc_stag;
1313 
1314 		struct {
1315 			struct irdma_sc_dev *dev;
1316 			struct irdma_mw_alloc_info info;
1317 			u64 scratch;
1318 		} mw_alloc;
1319 
1320 		struct {
1321 			struct irdma_sc_dev *dev;
1322 			struct irdma_reg_ns_stag_info info;
1323 			u64 scratch;
1324 		} mr_reg_non_shared;
1325 
1326 		struct {
1327 			struct irdma_sc_dev *dev;
1328 			struct irdma_dealloc_stag_info info;
1329 			u64 scratch;
1330 		} dealloc_stag;
1331 
1332 		struct {
1333 			struct irdma_sc_cqp *cqp;
1334 			struct irdma_add_arp_cache_entry_info info;
1335 			u64 scratch;
1336 		} add_arp_cache_entry;
1337 
1338 		struct {
1339 			struct irdma_sc_cqp *cqp;
1340 			u64 scratch;
1341 			u16 arp_index;
1342 		} del_arp_cache_entry;
1343 
1344 		struct {
1345 			struct irdma_sc_cqp *cqp;
1346 			struct irdma_local_mac_entry_info info;
1347 			u64 scratch;
1348 		} add_local_mac_entry;
1349 
1350 		struct {
1351 			struct irdma_sc_cqp *cqp;
1352 			u64 scratch;
1353 			u8 entry_idx;
1354 			u8 ignore_ref_count;
1355 		} del_local_mac_entry;
1356 
1357 		struct {
1358 			struct irdma_sc_cqp *cqp;
1359 			u64 scratch;
1360 		} alloc_local_mac_entry;
1361 
1362 		struct {
1363 			struct irdma_sc_cqp *cqp;
1364 			struct irdma_cqp_manage_push_page_info info;
1365 			u64 scratch;
1366 		} manage_push_page;
1367 
1368 		struct {
1369 			struct irdma_sc_dev *dev;
1370 			struct irdma_upload_context_info info;
1371 			u64 scratch;
1372 		} qp_upload_context;
1373 
1374 		struct {
1375 			struct irdma_sc_dev *dev;
1376 			struct irdma_hmc_fcn_info info;
1377 			u64 scratch;
1378 		} manage_hmc_pm;
1379 
1380 		struct {
1381 			struct irdma_sc_ceq *ceq;
1382 			u64 scratch;
1383 		} ceq_create;
1384 
1385 		struct {
1386 			struct irdma_sc_ceq *ceq;
1387 			u64 scratch;
1388 		} ceq_destroy;
1389 
1390 		struct {
1391 			struct irdma_sc_aeq *aeq;
1392 			u64 scratch;
1393 		} aeq_create;
1394 
1395 		struct {
1396 			struct irdma_sc_aeq *aeq;
1397 			u64 scratch;
1398 		} aeq_destroy;
1399 
1400 		struct {
1401 			struct irdma_sc_qp *qp;
1402 			struct irdma_qp_flush_info info;
1403 			u64 scratch;
1404 		} qp_flush_wqes;
1405 
1406 		struct {
1407 			struct irdma_sc_qp *qp;
1408 			struct irdma_gen_ae_info info;
1409 			u64 scratch;
1410 		} gen_ae;
1411 
1412 		struct {
1413 			struct irdma_sc_cqp *cqp;
1414 			void *fpm_val_va;
1415 			u64 fpm_val_pa;
1416 			u8 hmc_fn_id;
1417 			u64 scratch;
1418 		} query_fpm_val;
1419 
1420 		struct {
1421 			struct irdma_sc_cqp *cqp;
1422 			void *fpm_val_va;
1423 			u64 fpm_val_pa;
1424 			u8 hmc_fn_id;
1425 			u64 scratch;
1426 		} commit_fpm_val;
1427 
1428 		struct {
1429 			struct irdma_sc_cqp *cqp;
1430 			struct irdma_apbvt_info info;
1431 			u64 scratch;
1432 		} manage_apbvt_entry;
1433 
1434 		struct {
1435 			struct irdma_sc_cqp *cqp;
1436 			struct irdma_qhash_table_info info;
1437 			u64 scratch;
1438 		} manage_qhash_table_entry;
1439 
1440 		struct {
1441 			struct irdma_sc_dev *dev;
1442 			struct irdma_update_sds_info info;
1443 			u64 scratch;
1444 		} update_pe_sds;
1445 
1446 		struct {
1447 			struct irdma_sc_cqp *cqp;
1448 			struct irdma_sc_qp *qp;
1449 			u64 scratch;
1450 		} suspend_resume;
1451 
1452 		struct {
1453 			struct irdma_sc_cqp *cqp;
1454 			struct irdma_ah_info info;
1455 			u64 scratch;
1456 		} ah_create;
1457 
1458 		struct {
1459 			struct irdma_sc_cqp *cqp;
1460 			struct irdma_ah_info info;
1461 			u64 scratch;
1462 		} ah_destroy;
1463 
1464 		struct {
1465 			struct irdma_sc_cqp *cqp;
1466 			struct irdma_mcast_grp_info info;
1467 			u64 scratch;
1468 		} mc_create;
1469 
1470 		struct {
1471 			struct irdma_sc_cqp *cqp;
1472 			struct irdma_mcast_grp_info info;
1473 			u64 scratch;
1474 		} mc_destroy;
1475 
1476 		struct {
1477 			struct irdma_sc_cqp *cqp;
1478 			struct irdma_mcast_grp_info info;
1479 			u64 scratch;
1480 		} mc_modify;
1481 
1482 		struct {
1483 			struct irdma_sc_cqp *cqp;
1484 			struct irdma_stats_inst_info info;
1485 			u64 scratch;
1486 		} stats_manage;
1487 
1488 		struct {
1489 			struct irdma_sc_cqp *cqp;
1490 			struct irdma_stats_gather_info info;
1491 			u64 scratch;
1492 		} stats_gather;
1493 
1494 		struct {
1495 			struct irdma_sc_cqp *cqp;
1496 			struct irdma_ws_node_info info;
1497 			u64 scratch;
1498 		} ws_node;
1499 
1500 		struct {
1501 			struct irdma_sc_cqp *cqp;
1502 			struct irdma_up_info info;
1503 			u64 scratch;
1504 		} up_map;
1505 
1506 		struct {
1507 			struct irdma_sc_cqp *cqp;
1508 			struct irdma_dma_mem query_buff_mem;
1509 			u64 scratch;
1510 		} query_rdma;
1511 	} u;
1512 };
1513 
1514 struct cqp_cmds_info {
1515 	struct list_head cqp_cmd_entry;
1516 	u8 cqp_cmd;
1517 	u8 post_sq;
1518 	struct cqp_info in;
1519 };
1520 
1521 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1522 					   u32 *wqe_idx);
1523 
1524 /**
1525  * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1526  * @cqp: struct for cqp hw
1527  * @scratch: private data for CQP WQE
1528  */
irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp * cqp,u64 scratch)1529 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1530 {
1531 	u32 wqe_idx;
1532 
1533 	return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1534 }
1535 #endif /* IRDMA_TYPE_H */
1536