1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #ifndef _GDMA_H
5 #define _GDMA_H
6
7 #include <linux/dma-mapping.h>
8 #include <linux/netdevice.h>
9
10 #include "shm_channel.h"
11
12 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
13 * them are naturally aligned and hence don't need __packed.
14 */
15
16 enum gdma_request_type {
17 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
18 GDMA_QUERY_MAX_RESOURCES = 2,
19 GDMA_LIST_DEVICES = 3,
20 GDMA_REGISTER_DEVICE = 4,
21 GDMA_DEREGISTER_DEVICE = 5,
22 GDMA_GENERATE_TEST_EQE = 10,
23 GDMA_CREATE_QUEUE = 12,
24 GDMA_DISABLE_QUEUE = 13,
25 GDMA_CREATE_DMA_REGION = 25,
26 GDMA_DMA_REGION_ADD_PAGES = 26,
27 GDMA_DESTROY_DMA_REGION = 27,
28 };
29
30 enum gdma_queue_type {
31 GDMA_INVALID_QUEUE,
32 GDMA_SQ,
33 GDMA_RQ,
34 GDMA_CQ,
35 GDMA_EQ,
36 };
37
38 enum gdma_work_request_flags {
39 GDMA_WR_NONE = 0,
40 GDMA_WR_OOB_IN_SGL = BIT(0),
41 GDMA_WR_PAD_BY_SGE0 = BIT(1),
42 };
43
44 enum gdma_eqe_type {
45 GDMA_EQE_COMPLETION = 3,
46 GDMA_EQE_TEST_EVENT = 64,
47 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
48 GDMA_EQE_HWC_INIT_DATA = 130,
49 GDMA_EQE_HWC_INIT_DONE = 131,
50 };
51
52 enum {
53 GDMA_DEVICE_NONE = 0,
54 GDMA_DEVICE_HWC = 1,
55 GDMA_DEVICE_MANA = 2,
56 };
57
58 struct gdma_resource {
59 /* Protect the bitmap */
60 spinlock_t lock;
61
62 /* The bitmap size in bits. */
63 u32 size;
64
65 /* The bitmap tracks the resources. */
66 unsigned long *map;
67 };
68
69 union gdma_doorbell_entry {
70 u64 as_uint64;
71
72 struct {
73 u64 id : 24;
74 u64 reserved : 8;
75 u64 tail_ptr : 31;
76 u64 arm : 1;
77 } cq;
78
79 struct {
80 u64 id : 24;
81 u64 wqe_cnt : 8;
82 u64 tail_ptr : 32;
83 } rq;
84
85 struct {
86 u64 id : 24;
87 u64 reserved : 8;
88 u64 tail_ptr : 32;
89 } sq;
90
91 struct {
92 u64 id : 16;
93 u64 reserved : 16;
94 u64 tail_ptr : 31;
95 u64 arm : 1;
96 } eq;
97 }; /* HW DATA */
98
99 struct gdma_msg_hdr {
100 u32 hdr_type;
101 u32 msg_type;
102 u16 msg_version;
103 u16 hwc_msg_id;
104 u32 msg_size;
105 }; /* HW DATA */
106
107 struct gdma_dev_id {
108 union {
109 struct {
110 u16 type;
111 u16 instance;
112 };
113
114 u32 as_uint32;
115 };
116 }; /* HW DATA */
117
118 struct gdma_req_hdr {
119 struct gdma_msg_hdr req;
120 struct gdma_msg_hdr resp; /* The expected response */
121 struct gdma_dev_id dev_id;
122 u32 activity_id;
123 }; /* HW DATA */
124
125 struct gdma_resp_hdr {
126 struct gdma_msg_hdr response;
127 struct gdma_dev_id dev_id;
128 u32 activity_id;
129 u32 status;
130 u32 reserved;
131 }; /* HW DATA */
132
133 struct gdma_general_req {
134 struct gdma_req_hdr hdr;
135 }; /* HW DATA */
136
137 #define GDMA_MESSAGE_V1 1
138
139 struct gdma_general_resp {
140 struct gdma_resp_hdr hdr;
141 }; /* HW DATA */
142
143 #define GDMA_STANDARD_HEADER_TYPE 0
144
mana_gd_init_req_hdr(struct gdma_req_hdr * hdr,u32 code,u32 req_size,u32 resp_size)145 static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
146 u32 req_size, u32 resp_size)
147 {
148 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
149 hdr->req.msg_type = code;
150 hdr->req.msg_version = GDMA_MESSAGE_V1;
151 hdr->req.msg_size = req_size;
152
153 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
154 hdr->resp.msg_type = code;
155 hdr->resp.msg_version = GDMA_MESSAGE_V1;
156 hdr->resp.msg_size = resp_size;
157 }
158
159 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
160 struct gdma_sge {
161 u64 address;
162 u32 mem_key;
163 u32 size;
164 }; /* HW DATA */
165
166 struct gdma_wqe_request {
167 struct gdma_sge *sgl;
168 u32 num_sge;
169
170 u32 inline_oob_size;
171 const void *inline_oob_data;
172
173 u32 flags;
174 u32 client_data_unit;
175 };
176
177 enum gdma_page_type {
178 GDMA_PAGE_TYPE_4K,
179 };
180
181 #define GDMA_INVALID_DMA_REGION 0
182
183 struct gdma_mem_info {
184 struct device *dev;
185
186 dma_addr_t dma_handle;
187 void *virt_addr;
188 u64 length;
189
190 /* Allocated by the PF driver */
191 u64 gdma_region;
192 };
193
194 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
195
196 struct gdma_dev {
197 struct gdma_context *gdma_context;
198
199 struct gdma_dev_id dev_id;
200
201 u32 pdid;
202 u32 doorbell;
203 u32 gpa_mkey;
204
205 /* GDMA driver specific pointer */
206 void *driver_data;
207 };
208
209 #define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
210
211 #define GDMA_CQE_SIZE 64
212 #define GDMA_EQE_SIZE 16
213 #define GDMA_MAX_SQE_SIZE 512
214 #define GDMA_MAX_RQE_SIZE 256
215
216 #define GDMA_COMP_DATA_SIZE 0x3C
217
218 #define GDMA_EVENT_DATA_SIZE 0xC
219
220 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
221 #define GDMA_WQE_BU_SIZE 32
222
223 #define INVALID_PDID UINT_MAX
224 #define INVALID_DOORBELL UINT_MAX
225 #define INVALID_MEM_KEY UINT_MAX
226 #define INVALID_QUEUE_ID UINT_MAX
227 #define INVALID_PCI_MSIX_INDEX UINT_MAX
228
229 struct gdma_comp {
230 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
231 u32 wq_num;
232 bool is_sq;
233 };
234
235 struct gdma_event {
236 u32 details[GDMA_EVENT_DATA_SIZE / 4];
237 u8 type;
238 };
239
240 struct gdma_queue;
241
242 struct mana_eq {
243 struct gdma_queue *eq;
244 };
245
246 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
247 struct gdma_event *e);
248
249 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
250
251 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
252 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
253 * driver increases the 'head' in BUs rather than in bytes, and notifies
254 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
255 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
256 *
257 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
258 * processed, the driver increases the 'tail' to indicate that WQEs have
259 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
260 *
261 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
262 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
263 * the owner bits mechanism to detect if the queue has become empty.
264 */
265 struct gdma_queue {
266 struct gdma_dev *gdma_dev;
267
268 enum gdma_queue_type type;
269 u32 id;
270
271 struct gdma_mem_info mem_info;
272
273 void *queue_mem_ptr;
274 u32 queue_size;
275
276 bool monitor_avl_buf;
277
278 u32 head;
279 u32 tail;
280
281 /* Extra fields specific to EQ/CQ. */
282 union {
283 struct {
284 bool disable_needed;
285
286 gdma_eq_callback *callback;
287 void *context;
288
289 unsigned int msix_index;
290
291 u32 log2_throttle_limit;
292 } eq;
293
294 struct {
295 gdma_cq_callback *callback;
296 void *context;
297
298 struct gdma_queue *parent; /* For CQ/EQ relationship */
299 } cq;
300 };
301 };
302
303 struct gdma_queue_spec {
304 enum gdma_queue_type type;
305 bool monitor_avl_buf;
306 unsigned int queue_size;
307
308 /* Extra fields specific to EQ/CQ. */
309 union {
310 struct {
311 gdma_eq_callback *callback;
312 void *context;
313
314 unsigned long log2_throttle_limit;
315 } eq;
316
317 struct {
318 gdma_cq_callback *callback;
319 void *context;
320
321 struct gdma_queue *parent_eq;
322
323 } cq;
324 };
325 };
326
327 #define MANA_IRQ_NAME_SZ 32
328
329 struct gdma_irq_context {
330 void (*handler)(void *arg);
331 void *arg;
332 char name[MANA_IRQ_NAME_SZ];
333 };
334
335 struct gdma_context {
336 struct device *dev;
337
338 /* Per-vPort max number of queues */
339 unsigned int max_num_queues;
340 unsigned int max_num_msix;
341 unsigned int num_msix_usable;
342 struct gdma_resource msix_resource;
343 struct gdma_irq_context *irq_contexts;
344
345 /* This maps a CQ index to the queue structure. */
346 unsigned int max_num_cqs;
347 struct gdma_queue **cq_table;
348
349 /* Protect eq_test_event and test_event_eq_id */
350 struct mutex eq_test_event_mutex;
351 struct completion eq_test_event;
352 u32 test_event_eq_id;
353
354 bool is_pf;
355 void __iomem *bar0_va;
356 void __iomem *shm_base;
357 void __iomem *db_page_base;
358 u32 db_page_size;
359
360 /* Shared memory chanenl (used to bootstrap HWC) */
361 struct shm_channel shm_channel;
362
363 /* Hardware communication channel (HWC) */
364 struct gdma_dev hwc;
365
366 /* Azure network adapter */
367 struct gdma_dev mana;
368 };
369
370 #define MAX_NUM_GDMA_DEVICES 4
371
mana_gd_is_mana(struct gdma_dev * gd)372 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
373 {
374 return gd->dev_id.type == GDMA_DEVICE_MANA;
375 }
376
mana_gd_is_hwc(struct gdma_dev * gd)377 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
378 {
379 return gd->dev_id.type == GDMA_DEVICE_HWC;
380 }
381
382 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
383 u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
384
385 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
386
387 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
388 const struct gdma_queue_spec *spec,
389 struct gdma_queue **queue_ptr);
390
391 int mana_gd_create_mana_eq(struct gdma_dev *gd,
392 const struct gdma_queue_spec *spec,
393 struct gdma_queue **queue_ptr);
394
395 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
396 const struct gdma_queue_spec *spec,
397 struct gdma_queue **queue_ptr);
398
399 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
400
401 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
402
403 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
404
405 struct gdma_wqe {
406 u32 reserved :24;
407 u32 last_vbytes :8;
408
409 union {
410 u32 flags;
411
412 struct {
413 u32 num_sge :8;
414 u32 inline_oob_size_div4:3;
415 u32 client_oob_in_sgl :1;
416 u32 reserved1 :4;
417 u32 client_data_unit :14;
418 u32 reserved2 :2;
419 };
420 };
421 }; /* HW DATA */
422
423 #define INLINE_OOB_SMALL_SIZE 8
424 #define INLINE_OOB_LARGE_SIZE 24
425
426 #define MAX_TX_WQE_SIZE 512
427 #define MAX_RX_WQE_SIZE 256
428
429 struct gdma_cqe {
430 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
431
432 union {
433 u32 as_uint32;
434
435 struct {
436 u32 wq_num : 24;
437 u32 is_sq : 1;
438 u32 reserved : 4;
439 u32 owner_bits : 3;
440 };
441 } cqe_info;
442 }; /* HW DATA */
443
444 #define GDMA_CQE_OWNER_BITS 3
445
446 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
447
448 #define SET_ARM_BIT 1
449
450 #define GDMA_EQE_OWNER_BITS 3
451
452 union gdma_eqe_info {
453 u32 as_uint32;
454
455 struct {
456 u32 type : 8;
457 u32 reserved1 : 8;
458 u32 client_id : 2;
459 u32 reserved2 : 11;
460 u32 owner_bits : 3;
461 };
462 }; /* HW DATA */
463
464 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
465 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
466
467 struct gdma_eqe {
468 u32 details[GDMA_EVENT_DATA_SIZE / 4];
469 u32 eqe_info;
470 }; /* HW DATA */
471
472 #define GDMA_REG_DB_PAGE_OFFSET 8
473 #define GDMA_REG_DB_PAGE_SIZE 0x10
474 #define GDMA_REG_SHM_OFFSET 0x18
475
476 #define GDMA_PF_REG_DB_PAGE_SIZE 0xD0
477 #define GDMA_PF_REG_DB_PAGE_OFF 0xC8
478 #define GDMA_PF_REG_SHM_OFF 0x70
479
480 #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108
481
482 #define MANA_PF_DEVICE_ID 0x00B9
483 #define MANA_VF_DEVICE_ID 0x00BA
484
485 struct gdma_posted_wqe_info {
486 u32 wqe_size_in_bu;
487 };
488
489 /* GDMA_GENERATE_TEST_EQE */
490 struct gdma_generate_test_event_req {
491 struct gdma_req_hdr hdr;
492 u32 queue_index;
493 }; /* HW DATA */
494
495 /* GDMA_VERIFY_VF_DRIVER_VERSION */
496 enum {
497 GDMA_PROTOCOL_V1 = 1,
498 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
499 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
500 };
501
502 #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
503
504 /* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
505 * so the driver is able to reliably support features like busy_poll.
506 */
507 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
508
509 #define GDMA_DRV_CAP_FLAGS1 \
510 (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
511 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX)
512
513 #define GDMA_DRV_CAP_FLAGS2 0
514
515 #define GDMA_DRV_CAP_FLAGS3 0
516
517 #define GDMA_DRV_CAP_FLAGS4 0
518
519 struct gdma_verify_ver_req {
520 struct gdma_req_hdr hdr;
521
522 /* Mandatory fields required for protocol establishment */
523 u64 protocol_ver_min;
524 u64 protocol_ver_max;
525
526 /* Gdma Driver Capability Flags */
527 u64 gd_drv_cap_flags1;
528 u64 gd_drv_cap_flags2;
529 u64 gd_drv_cap_flags3;
530 u64 gd_drv_cap_flags4;
531
532 /* Advisory fields */
533 u64 drv_ver;
534 u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
535 u32 reserved;
536 u32 os_ver_major;
537 u32 os_ver_minor;
538 u32 os_ver_build;
539 u32 os_ver_platform;
540 u64 reserved_2;
541 u8 os_ver_str1[128];
542 u8 os_ver_str2[128];
543 u8 os_ver_str3[128];
544 u8 os_ver_str4[128];
545 }; /* HW DATA */
546
547 struct gdma_verify_ver_resp {
548 struct gdma_resp_hdr hdr;
549 u64 gdma_protocol_ver;
550 u64 pf_cap_flags1;
551 u64 pf_cap_flags2;
552 u64 pf_cap_flags3;
553 u64 pf_cap_flags4;
554 }; /* HW DATA */
555
556 /* GDMA_QUERY_MAX_RESOURCES */
557 struct gdma_query_max_resources_resp {
558 struct gdma_resp_hdr hdr;
559 u32 status;
560 u32 max_sq;
561 u32 max_rq;
562 u32 max_cq;
563 u32 max_eq;
564 u32 max_db;
565 u32 max_mst;
566 u32 max_cq_mod_ctx;
567 u32 max_mod_cq;
568 u32 max_msix;
569 }; /* HW DATA */
570
571 /* GDMA_LIST_DEVICES */
572 struct gdma_list_devices_resp {
573 struct gdma_resp_hdr hdr;
574 u32 num_of_devs;
575 u32 reserved;
576 struct gdma_dev_id devs[64];
577 }; /* HW DATA */
578
579 /* GDMA_REGISTER_DEVICE */
580 struct gdma_register_device_resp {
581 struct gdma_resp_hdr hdr;
582 u32 pdid;
583 u32 gpa_mkey;
584 u32 db_id;
585 }; /* HW DATA */
586
587 /* GDMA_CREATE_QUEUE */
588 struct gdma_create_queue_req {
589 struct gdma_req_hdr hdr;
590 u32 type;
591 u32 reserved1;
592 u32 pdid;
593 u32 doolbell_id;
594 u64 gdma_region;
595 u32 reserved2;
596 u32 queue_size;
597 u32 log2_throttle_limit;
598 u32 eq_pci_msix_index;
599 u32 cq_mod_ctx_id;
600 u32 cq_parent_eq_id;
601 u8 rq_drop_on_overrun;
602 u8 rq_err_on_wqe_overflow;
603 u8 rq_chain_rec_wqes;
604 u8 sq_hw_db;
605 u32 reserved3;
606 }; /* HW DATA */
607
608 struct gdma_create_queue_resp {
609 struct gdma_resp_hdr hdr;
610 u32 queue_index;
611 }; /* HW DATA */
612
613 /* GDMA_DISABLE_QUEUE */
614 struct gdma_disable_queue_req {
615 struct gdma_req_hdr hdr;
616 u32 type;
617 u32 queue_index;
618 u32 alloc_res_id_on_creation;
619 }; /* HW DATA */
620
621 /* GDMA_CREATE_DMA_REGION */
622 struct gdma_create_dma_region_req {
623 struct gdma_req_hdr hdr;
624
625 /* The total size of the DMA region */
626 u64 length;
627
628 /* The offset in the first page */
629 u32 offset_in_page;
630
631 /* enum gdma_page_type */
632 u32 gdma_page_type;
633
634 /* The total number of pages */
635 u32 page_count;
636
637 /* If page_addr_list_len is smaller than page_count,
638 * the remaining page addresses will be added via the
639 * message GDMA_DMA_REGION_ADD_PAGES.
640 */
641 u32 page_addr_list_len;
642 u64 page_addr_list[];
643 }; /* HW DATA */
644
645 struct gdma_create_dma_region_resp {
646 struct gdma_resp_hdr hdr;
647 u64 gdma_region;
648 }; /* HW DATA */
649
650 /* GDMA_DMA_REGION_ADD_PAGES */
651 struct gdma_dma_region_add_pages_req {
652 struct gdma_req_hdr hdr;
653
654 u64 gdma_region;
655
656 u32 page_addr_list_len;
657 u32 reserved3;
658
659 u64 page_addr_list[];
660 }; /* HW DATA */
661
662 /* GDMA_DESTROY_DMA_REGION */
663 struct gdma_destroy_dma_region_req {
664 struct gdma_req_hdr hdr;
665
666 u64 gdma_region;
667 }; /* HW DATA */
668
669 int mana_gd_verify_vf_version(struct pci_dev *pdev);
670
671 int mana_gd_register_device(struct gdma_dev *gd);
672 int mana_gd_deregister_device(struct gdma_dev *gd);
673
674 int mana_gd_post_work_request(struct gdma_queue *wq,
675 const struct gdma_wqe_request *wqe_req,
676 struct gdma_posted_wqe_info *wqe_info);
677
678 int mana_gd_post_and_ring(struct gdma_queue *queue,
679 const struct gdma_wqe_request *wqe,
680 struct gdma_posted_wqe_info *wqe_info);
681
682 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
683 void mana_gd_free_res_map(struct gdma_resource *r);
684
685 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
686 struct gdma_queue *queue);
687
688 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
689 struct gdma_mem_info *gmi);
690
691 void mana_gd_free_memory(struct gdma_mem_info *gmi);
692
693 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
694 u32 resp_len, void *resp);
695 #endif /* _GDMA_H */
696