1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2021, Microsoft Corporation. */
3 
4 #ifndef _GDMA_H
5 #define _GDMA_H
6 
7 #include <linux/dma-mapping.h>
8 #include <linux/netdevice.h>
9 
10 #include "shm_channel.h"
11 
12 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
13  * them are naturally aligned and hence don't need __packed.
14  */
15 
16 enum gdma_request_type {
17 	GDMA_VERIFY_VF_DRIVER_VERSION	= 1,
18 	GDMA_QUERY_MAX_RESOURCES	= 2,
19 	GDMA_LIST_DEVICES		= 3,
20 	GDMA_REGISTER_DEVICE		= 4,
21 	GDMA_DEREGISTER_DEVICE		= 5,
22 	GDMA_GENERATE_TEST_EQE		= 10,
23 	GDMA_CREATE_QUEUE		= 12,
24 	GDMA_DISABLE_QUEUE		= 13,
25 	GDMA_CREATE_DMA_REGION		= 25,
26 	GDMA_DMA_REGION_ADD_PAGES	= 26,
27 	GDMA_DESTROY_DMA_REGION		= 27,
28 };
29 
30 enum gdma_queue_type {
31 	GDMA_INVALID_QUEUE,
32 	GDMA_SQ,
33 	GDMA_RQ,
34 	GDMA_CQ,
35 	GDMA_EQ,
36 };
37 
38 enum gdma_work_request_flags {
39 	GDMA_WR_NONE			= 0,
40 	GDMA_WR_OOB_IN_SGL		= BIT(0),
41 	GDMA_WR_PAD_BY_SGE0		= BIT(1),
42 };
43 
44 enum gdma_eqe_type {
45 	GDMA_EQE_COMPLETION		= 3,
46 	GDMA_EQE_TEST_EVENT		= 64,
47 	GDMA_EQE_HWC_INIT_EQ_ID_DB	= 129,
48 	GDMA_EQE_HWC_INIT_DATA		= 130,
49 	GDMA_EQE_HWC_INIT_DONE		= 131,
50 };
51 
52 enum {
53 	GDMA_DEVICE_NONE	= 0,
54 	GDMA_DEVICE_HWC		= 1,
55 	GDMA_DEVICE_MANA	= 2,
56 };
57 
58 struct gdma_resource {
59 	/* Protect the bitmap */
60 	spinlock_t lock;
61 
62 	/* The bitmap size in bits. */
63 	u32 size;
64 
65 	/* The bitmap tracks the resources. */
66 	unsigned long *map;
67 };
68 
69 union gdma_doorbell_entry {
70 	u64	as_uint64;
71 
72 	struct {
73 		u64 id		: 24;
74 		u64 reserved	: 8;
75 		u64 tail_ptr	: 31;
76 		u64 arm		: 1;
77 	} cq;
78 
79 	struct {
80 		u64 id		: 24;
81 		u64 wqe_cnt	: 8;
82 		u64 tail_ptr	: 32;
83 	} rq;
84 
85 	struct {
86 		u64 id		: 24;
87 		u64 reserved	: 8;
88 		u64 tail_ptr	: 32;
89 	} sq;
90 
91 	struct {
92 		u64 id		: 16;
93 		u64 reserved	: 16;
94 		u64 tail_ptr	: 31;
95 		u64 arm		: 1;
96 	} eq;
97 }; /* HW DATA */
98 
99 struct gdma_msg_hdr {
100 	u32 hdr_type;
101 	u32 msg_type;
102 	u16 msg_version;
103 	u16 hwc_msg_id;
104 	u32 msg_size;
105 }; /* HW DATA */
106 
107 struct gdma_dev_id {
108 	union {
109 		struct {
110 			u16 type;
111 			u16 instance;
112 		};
113 
114 		u32 as_uint32;
115 	};
116 }; /* HW DATA */
117 
118 struct gdma_req_hdr {
119 	struct gdma_msg_hdr req;
120 	struct gdma_msg_hdr resp; /* The expected response */
121 	struct gdma_dev_id dev_id;
122 	u32 activity_id;
123 }; /* HW DATA */
124 
125 struct gdma_resp_hdr {
126 	struct gdma_msg_hdr response;
127 	struct gdma_dev_id dev_id;
128 	u32 activity_id;
129 	u32 status;
130 	u32 reserved;
131 }; /* HW DATA */
132 
133 struct gdma_general_req {
134 	struct gdma_req_hdr hdr;
135 }; /* HW DATA */
136 
137 #define GDMA_MESSAGE_V1 1
138 
139 struct gdma_general_resp {
140 	struct gdma_resp_hdr hdr;
141 }; /* HW DATA */
142 
143 #define GDMA_STANDARD_HEADER_TYPE 0
144 
mana_gd_init_req_hdr(struct gdma_req_hdr * hdr,u32 code,u32 req_size,u32 resp_size)145 static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
146 					u32 req_size, u32 resp_size)
147 {
148 	hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
149 	hdr->req.msg_type = code;
150 	hdr->req.msg_version = GDMA_MESSAGE_V1;
151 	hdr->req.msg_size = req_size;
152 
153 	hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
154 	hdr->resp.msg_type = code;
155 	hdr->resp.msg_version = GDMA_MESSAGE_V1;
156 	hdr->resp.msg_size = resp_size;
157 }
158 
159 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
160 struct gdma_sge {
161 	u64 address;
162 	u32 mem_key;
163 	u32 size;
164 }; /* HW DATA */
165 
166 struct gdma_wqe_request {
167 	struct gdma_sge *sgl;
168 	u32 num_sge;
169 
170 	u32 inline_oob_size;
171 	const void *inline_oob_data;
172 
173 	u32 flags;
174 	u32 client_data_unit;
175 };
176 
177 enum gdma_page_type {
178 	GDMA_PAGE_TYPE_4K,
179 };
180 
181 #define GDMA_INVALID_DMA_REGION 0
182 
183 struct gdma_mem_info {
184 	struct device *dev;
185 
186 	dma_addr_t dma_handle;
187 	void *virt_addr;
188 	u64 length;
189 
190 	/* Allocated by the PF driver */
191 	u64 gdma_region;
192 };
193 
194 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
195 
196 struct gdma_dev {
197 	struct gdma_context *gdma_context;
198 
199 	struct gdma_dev_id dev_id;
200 
201 	u32 pdid;
202 	u32 doorbell;
203 	u32 gpa_mkey;
204 
205 	/* GDMA driver specific pointer */
206 	void *driver_data;
207 };
208 
209 #define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
210 
211 #define GDMA_CQE_SIZE 64
212 #define GDMA_EQE_SIZE 16
213 #define GDMA_MAX_SQE_SIZE 512
214 #define GDMA_MAX_RQE_SIZE 256
215 
216 #define GDMA_COMP_DATA_SIZE 0x3C
217 
218 #define GDMA_EVENT_DATA_SIZE 0xC
219 
220 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
221 #define GDMA_WQE_BU_SIZE 32
222 
223 #define INVALID_PDID		UINT_MAX
224 #define INVALID_DOORBELL	UINT_MAX
225 #define INVALID_MEM_KEY		UINT_MAX
226 #define INVALID_QUEUE_ID	UINT_MAX
227 #define INVALID_PCI_MSIX_INDEX  UINT_MAX
228 
229 struct gdma_comp {
230 	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
231 	u32 wq_num;
232 	bool is_sq;
233 };
234 
235 struct gdma_event {
236 	u32 details[GDMA_EVENT_DATA_SIZE / 4];
237 	u8  type;
238 };
239 
240 struct gdma_queue;
241 
242 struct mana_eq {
243 	struct gdma_queue *eq;
244 };
245 
246 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
247 			      struct gdma_event *e);
248 
249 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
250 
251 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
252  * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
253  * driver increases the 'head' in BUs rather than in bytes, and notifies
254  * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
255  * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
256  *
257  * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
258  * processed, the driver increases the 'tail' to indicate that WQEs have
259  * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
260  *
261  * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
262  * that the EQ/CQ is big enough so they can't overflow, and the driver uses
263  * the owner bits mechanism to detect if the queue has become empty.
264  */
265 struct gdma_queue {
266 	struct gdma_dev *gdma_dev;
267 
268 	enum gdma_queue_type type;
269 	u32 id;
270 
271 	struct gdma_mem_info mem_info;
272 
273 	void *queue_mem_ptr;
274 	u32 queue_size;
275 
276 	bool monitor_avl_buf;
277 
278 	u32 head;
279 	u32 tail;
280 
281 	/* Extra fields specific to EQ/CQ. */
282 	union {
283 		struct {
284 			bool disable_needed;
285 
286 			gdma_eq_callback *callback;
287 			void *context;
288 
289 			unsigned int msix_index;
290 
291 			u32 log2_throttle_limit;
292 		} eq;
293 
294 		struct {
295 			gdma_cq_callback *callback;
296 			void *context;
297 
298 			struct gdma_queue *parent; /* For CQ/EQ relationship */
299 		} cq;
300 	};
301 };
302 
303 struct gdma_queue_spec {
304 	enum gdma_queue_type type;
305 	bool monitor_avl_buf;
306 	unsigned int queue_size;
307 
308 	/* Extra fields specific to EQ/CQ. */
309 	union {
310 		struct {
311 			gdma_eq_callback *callback;
312 			void *context;
313 
314 			unsigned long log2_throttle_limit;
315 		} eq;
316 
317 		struct {
318 			gdma_cq_callback *callback;
319 			void *context;
320 
321 			struct gdma_queue *parent_eq;
322 
323 		} cq;
324 	};
325 };
326 
327 #define MANA_IRQ_NAME_SZ 32
328 
329 struct gdma_irq_context {
330 	void (*handler)(void *arg);
331 	void *arg;
332 	char name[MANA_IRQ_NAME_SZ];
333 };
334 
335 struct gdma_context {
336 	struct device		*dev;
337 
338 	/* Per-vPort max number of queues */
339 	unsigned int		max_num_queues;
340 	unsigned int		max_num_msix;
341 	unsigned int		num_msix_usable;
342 	struct gdma_resource	msix_resource;
343 	struct gdma_irq_context	*irq_contexts;
344 
345 	/* This maps a CQ index to the queue structure. */
346 	unsigned int		max_num_cqs;
347 	struct gdma_queue	**cq_table;
348 
349 	/* Protect eq_test_event and test_event_eq_id  */
350 	struct mutex		eq_test_event_mutex;
351 	struct completion	eq_test_event;
352 	u32			test_event_eq_id;
353 
354 	bool			is_pf;
355 	void __iomem		*bar0_va;
356 	void __iomem		*shm_base;
357 	void __iomem		*db_page_base;
358 	u32 db_page_size;
359 
360 	/* Shared memory chanenl (used to bootstrap HWC) */
361 	struct shm_channel	shm_channel;
362 
363 	/* Hardware communication channel (HWC) */
364 	struct gdma_dev		hwc;
365 
366 	/* Azure network adapter */
367 	struct gdma_dev		mana;
368 };
369 
370 #define MAX_NUM_GDMA_DEVICES	4
371 
mana_gd_is_mana(struct gdma_dev * gd)372 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
373 {
374 	return gd->dev_id.type == GDMA_DEVICE_MANA;
375 }
376 
mana_gd_is_hwc(struct gdma_dev * gd)377 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
378 {
379 	return gd->dev_id.type == GDMA_DEVICE_HWC;
380 }
381 
382 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
383 u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
384 
385 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
386 
387 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
388 			     const struct gdma_queue_spec *spec,
389 			     struct gdma_queue **queue_ptr);
390 
391 int mana_gd_create_mana_eq(struct gdma_dev *gd,
392 			   const struct gdma_queue_spec *spec,
393 			   struct gdma_queue **queue_ptr);
394 
395 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
396 			      const struct gdma_queue_spec *spec,
397 			      struct gdma_queue **queue_ptr);
398 
399 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
400 
401 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
402 
403 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
404 
405 struct gdma_wqe {
406 	u32 reserved	:24;
407 	u32 last_vbytes	:8;
408 
409 	union {
410 		u32 flags;
411 
412 		struct {
413 			u32 num_sge		:8;
414 			u32 inline_oob_size_div4:3;
415 			u32 client_oob_in_sgl	:1;
416 			u32 reserved1		:4;
417 			u32 client_data_unit	:14;
418 			u32 reserved2		:2;
419 		};
420 	};
421 }; /* HW DATA */
422 
423 #define INLINE_OOB_SMALL_SIZE 8
424 #define INLINE_OOB_LARGE_SIZE 24
425 
426 #define MAX_TX_WQE_SIZE 512
427 #define MAX_RX_WQE_SIZE 256
428 
429 struct gdma_cqe {
430 	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
431 
432 	union {
433 		u32 as_uint32;
434 
435 		struct {
436 			u32 wq_num	: 24;
437 			u32 is_sq	: 1;
438 			u32 reserved	: 4;
439 			u32 owner_bits	: 3;
440 		};
441 	} cqe_info;
442 }; /* HW DATA */
443 
444 #define GDMA_CQE_OWNER_BITS 3
445 
446 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
447 
448 #define SET_ARM_BIT 1
449 
450 #define GDMA_EQE_OWNER_BITS 3
451 
452 union gdma_eqe_info {
453 	u32 as_uint32;
454 
455 	struct {
456 		u32 type	: 8;
457 		u32 reserved1	: 8;
458 		u32 client_id	: 2;
459 		u32 reserved2	: 11;
460 		u32 owner_bits	: 3;
461 	};
462 }; /* HW DATA */
463 
464 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
465 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
466 
467 struct gdma_eqe {
468 	u32 details[GDMA_EVENT_DATA_SIZE / 4];
469 	u32 eqe_info;
470 }; /* HW DATA */
471 
472 #define GDMA_REG_DB_PAGE_OFFSET	8
473 #define GDMA_REG_DB_PAGE_SIZE	0x10
474 #define GDMA_REG_SHM_OFFSET	0x18
475 
476 #define GDMA_PF_REG_DB_PAGE_SIZE	0xD0
477 #define GDMA_PF_REG_DB_PAGE_OFF		0xC8
478 #define GDMA_PF_REG_SHM_OFF		0x70
479 
480 #define GDMA_SRIOV_REG_CFG_BASE_OFF	0x108
481 
482 #define MANA_PF_DEVICE_ID 0x00B9
483 #define MANA_VF_DEVICE_ID 0x00BA
484 
485 struct gdma_posted_wqe_info {
486 	u32 wqe_size_in_bu;
487 };
488 
489 /* GDMA_GENERATE_TEST_EQE */
490 struct gdma_generate_test_event_req {
491 	struct gdma_req_hdr hdr;
492 	u32 queue_index;
493 }; /* HW DATA */
494 
495 /* GDMA_VERIFY_VF_DRIVER_VERSION */
496 enum {
497 	GDMA_PROTOCOL_V1	= 1,
498 	GDMA_PROTOCOL_FIRST	= GDMA_PROTOCOL_V1,
499 	GDMA_PROTOCOL_LAST	= GDMA_PROTOCOL_V1,
500 };
501 
502 #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
503 
504 /* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
505  * so the driver is able to reliably support features like busy_poll.
506  */
507 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
508 
509 #define GDMA_DRV_CAP_FLAGS1 \
510 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
511 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX)
512 
513 #define GDMA_DRV_CAP_FLAGS2 0
514 
515 #define GDMA_DRV_CAP_FLAGS3 0
516 
517 #define GDMA_DRV_CAP_FLAGS4 0
518 
519 struct gdma_verify_ver_req {
520 	struct gdma_req_hdr hdr;
521 
522 	/* Mandatory fields required for protocol establishment */
523 	u64 protocol_ver_min;
524 	u64 protocol_ver_max;
525 
526 	/* Gdma Driver Capability Flags */
527 	u64 gd_drv_cap_flags1;
528 	u64 gd_drv_cap_flags2;
529 	u64 gd_drv_cap_flags3;
530 	u64 gd_drv_cap_flags4;
531 
532 	/* Advisory fields */
533 	u64 drv_ver;
534 	u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
535 	u32 reserved;
536 	u32 os_ver_major;
537 	u32 os_ver_minor;
538 	u32 os_ver_build;
539 	u32 os_ver_platform;
540 	u64 reserved_2;
541 	u8 os_ver_str1[128];
542 	u8 os_ver_str2[128];
543 	u8 os_ver_str3[128];
544 	u8 os_ver_str4[128];
545 }; /* HW DATA */
546 
547 struct gdma_verify_ver_resp {
548 	struct gdma_resp_hdr hdr;
549 	u64 gdma_protocol_ver;
550 	u64 pf_cap_flags1;
551 	u64 pf_cap_flags2;
552 	u64 pf_cap_flags3;
553 	u64 pf_cap_flags4;
554 }; /* HW DATA */
555 
556 /* GDMA_QUERY_MAX_RESOURCES */
557 struct gdma_query_max_resources_resp {
558 	struct gdma_resp_hdr hdr;
559 	u32 status;
560 	u32 max_sq;
561 	u32 max_rq;
562 	u32 max_cq;
563 	u32 max_eq;
564 	u32 max_db;
565 	u32 max_mst;
566 	u32 max_cq_mod_ctx;
567 	u32 max_mod_cq;
568 	u32 max_msix;
569 }; /* HW DATA */
570 
571 /* GDMA_LIST_DEVICES */
572 struct gdma_list_devices_resp {
573 	struct gdma_resp_hdr hdr;
574 	u32 num_of_devs;
575 	u32 reserved;
576 	struct gdma_dev_id devs[64];
577 }; /* HW DATA */
578 
579 /* GDMA_REGISTER_DEVICE */
580 struct gdma_register_device_resp {
581 	struct gdma_resp_hdr hdr;
582 	u32 pdid;
583 	u32 gpa_mkey;
584 	u32 db_id;
585 }; /* HW DATA */
586 
587 /* GDMA_CREATE_QUEUE */
588 struct gdma_create_queue_req {
589 	struct gdma_req_hdr hdr;
590 	u32 type;
591 	u32 reserved1;
592 	u32 pdid;
593 	u32 doolbell_id;
594 	u64 gdma_region;
595 	u32 reserved2;
596 	u32 queue_size;
597 	u32 log2_throttle_limit;
598 	u32 eq_pci_msix_index;
599 	u32 cq_mod_ctx_id;
600 	u32 cq_parent_eq_id;
601 	u8  rq_drop_on_overrun;
602 	u8  rq_err_on_wqe_overflow;
603 	u8  rq_chain_rec_wqes;
604 	u8  sq_hw_db;
605 	u32 reserved3;
606 }; /* HW DATA */
607 
608 struct gdma_create_queue_resp {
609 	struct gdma_resp_hdr hdr;
610 	u32 queue_index;
611 }; /* HW DATA */
612 
613 /* GDMA_DISABLE_QUEUE */
614 struct gdma_disable_queue_req {
615 	struct gdma_req_hdr hdr;
616 	u32 type;
617 	u32 queue_index;
618 	u32 alloc_res_id_on_creation;
619 }; /* HW DATA */
620 
621 /* GDMA_CREATE_DMA_REGION */
622 struct gdma_create_dma_region_req {
623 	struct gdma_req_hdr hdr;
624 
625 	/* The total size of the DMA region */
626 	u64 length;
627 
628 	/* The offset in the first page */
629 	u32 offset_in_page;
630 
631 	/* enum gdma_page_type */
632 	u32 gdma_page_type;
633 
634 	/* The total number of pages */
635 	u32 page_count;
636 
637 	/* If page_addr_list_len is smaller than page_count,
638 	 * the remaining page addresses will be added via the
639 	 * message GDMA_DMA_REGION_ADD_PAGES.
640 	 */
641 	u32 page_addr_list_len;
642 	u64 page_addr_list[];
643 }; /* HW DATA */
644 
645 struct gdma_create_dma_region_resp {
646 	struct gdma_resp_hdr hdr;
647 	u64 gdma_region;
648 }; /* HW DATA */
649 
650 /* GDMA_DMA_REGION_ADD_PAGES */
651 struct gdma_dma_region_add_pages_req {
652 	struct gdma_req_hdr hdr;
653 
654 	u64 gdma_region;
655 
656 	u32 page_addr_list_len;
657 	u32 reserved3;
658 
659 	u64 page_addr_list[];
660 }; /* HW DATA */
661 
662 /* GDMA_DESTROY_DMA_REGION */
663 struct gdma_destroy_dma_region_req {
664 	struct gdma_req_hdr hdr;
665 
666 	u64 gdma_region;
667 }; /* HW DATA */
668 
669 int mana_gd_verify_vf_version(struct pci_dev *pdev);
670 
671 int mana_gd_register_device(struct gdma_dev *gd);
672 int mana_gd_deregister_device(struct gdma_dev *gd);
673 
674 int mana_gd_post_work_request(struct gdma_queue *wq,
675 			      const struct gdma_wqe_request *wqe_req,
676 			      struct gdma_posted_wqe_info *wqe_info);
677 
678 int mana_gd_post_and_ring(struct gdma_queue *queue,
679 			  const struct gdma_wqe_request *wqe,
680 			  struct gdma_posted_wqe_info *wqe_info);
681 
682 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
683 void mana_gd_free_res_map(struct gdma_resource *r);
684 
685 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
686 			      struct gdma_queue *queue);
687 
688 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
689 			 struct gdma_mem_info *gmi);
690 
691 void mana_gd_free_memory(struct gdma_mem_info *gmi);
692 
693 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
694 			 u32 resp_len, void *resp);
695 #endif /* _GDMA_H */
696