1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/errno.h>
10 #include <linux/pci.h>
11 #include <linux/device.h>
12 #include <linux/slab.h>
13 #include <linux/vmalloc.h>
14 #include <linux/spinlock.h>
15 #include <linux/sizes.h>
16 #include <linux/atomic.h>
17 #include <linux/log2.h>
18 #include <linux/io.h>
19 #include <linux/completion.h>
20 #include <linux/err.h>
21 #include <asm/byteorder.h>
22 #include <asm/barrier.h>
23 
24 #include "hinic_common.h"
25 #include "hinic_hw_if.h"
26 #include "hinic_hw_eqs.h"
27 #include "hinic_hw_mgmt.h"
28 #include "hinic_hw_wqe.h"
29 #include "hinic_hw_wq.h"
30 #include "hinic_hw_cmdq.h"
31 #include "hinic_hw_io.h"
32 #include "hinic_hw_dev.h"
33 
34 #define CMDQ_CEQE_TYPE_SHIFT                    0
35 
36 #define CMDQ_CEQE_TYPE_MASK                     0x7
37 
38 #define CMDQ_CEQE_GET(val, member)              \
39 			(((val) >> CMDQ_CEQE_##member##_SHIFT) \
40 			 & CMDQ_CEQE_##member##_MASK)
41 
42 #define CMDQ_WQE_ERRCODE_VAL_SHIFT              20
43 
44 #define CMDQ_WQE_ERRCODE_VAL_MASK               0xF
45 
46 #define CMDQ_WQE_ERRCODE_GET(val, member)       \
47 			(((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \
48 			 & CMDQ_WQE_ERRCODE_##member##_MASK)
49 
50 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
51 
52 #define CMDQ_DB_ADDR(db_base, pi)       ((db_base) + CMDQ_DB_PI_OFF(pi))
53 
54 #define CMDQ_WQE_HEADER(wqe)            ((struct hinic_cmdq_header *)(wqe))
55 
56 #define CMDQ_WQE_COMPLETED(ctrl_info)   \
57 			HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
58 
59 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
60 
61 #define CMDQ_DB_OFF                     SZ_2K
62 
63 #define CMDQ_WQEBB_SIZE                 64
64 #define CMDQ_WQE_SIZE                   64
65 #define CMDQ_DEPTH                      SZ_4K
66 
67 #define CMDQ_WQ_PAGE_SIZE               SZ_256K
68 
69 #define WQE_LCMD_SIZE                   64
70 #define WQE_SCMD_SIZE                   64
71 
72 #define COMPLETE_LEN                    3
73 
74 #define CMDQ_TIMEOUT                    1000
75 
76 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
77 
78 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
79 					     struct hinic_cmdqs, cmdq[0])
80 
81 #define cmdqs_to_func_to_io(cmdqs)      container_of(cmdqs, \
82 						     struct hinic_func_to_io, \
83 						     cmdqs)
84 
85 enum cmdq_wqe_type {
86 	WQE_LCMD_TYPE = 0,
87 	WQE_SCMD_TYPE = 1,
88 };
89 
90 enum completion_format {
91 	COMPLETE_DIRECT = 0,
92 	COMPLETE_SGE    = 1,
93 };
94 
95 enum data_format {
96 	DATA_SGE        = 0,
97 	DATA_DIRECT     = 1,
98 };
99 
100 enum bufdesc_len {
101 	BUFDESC_LCMD_LEN = 2,   /* 16 bytes - 2(8 byte unit) */
102 	BUFDESC_SCMD_LEN = 3,   /* 24 bytes - 3(8 byte unit) */
103 };
104 
105 enum ctrl_sect_len {
106 	CTRL_SECT_LEN        = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */
107 	CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */
108 };
109 
110 enum cmdq_scmd_type {
111 	CMDQ_SET_ARM_CMD = 2,
112 };
113 
114 enum cmdq_cmd_type {
115 	CMDQ_CMD_SYNC_DIRECT_RESP = 0,
116 	CMDQ_CMD_SYNC_SGE_RESP    = 1,
117 };
118 
119 enum completion_request {
120 	NO_CEQ  = 0,
121 	CEQ_SET = 1,
122 };
123 
124 /**
125  * hinic_alloc_cmdq_buf - alloc buffer for sending command
126  * @cmdqs: the cmdqs
127  * @cmdq_buf: the buffer returned in this struct
128  *
129  * Return 0 - Success, negative - Failure
130  **/
hinic_alloc_cmdq_buf(struct hinic_cmdqs * cmdqs,struct hinic_cmdq_buf * cmdq_buf)131 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
132 			 struct hinic_cmdq_buf *cmdq_buf)
133 {
134 	struct hinic_hwif *hwif = cmdqs->hwif;
135 	struct pci_dev *pdev = hwif->pdev;
136 
137 	cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
138 				       &cmdq_buf->dma_addr);
139 	if (!cmdq_buf->buf) {
140 		dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n");
141 		return -ENOMEM;
142 	}
143 
144 	return 0;
145 }
146 
147 /**
148  * hinic_free_cmdq_buf - free buffer
149  * @cmdqs: the cmdqs
150  * @cmdq_buf: the buffer to free that is in this struct
151  **/
hinic_free_cmdq_buf(struct hinic_cmdqs * cmdqs,struct hinic_cmdq_buf * cmdq_buf)152 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
153 			 struct hinic_cmdq_buf *cmdq_buf)
154 {
155 	dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
156 }
157 
cmdq_wqe_size_from_bdlen(enum bufdesc_len len)158 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len)
159 {
160 	unsigned int wqe_size = 0;
161 
162 	switch (len) {
163 	case BUFDESC_LCMD_LEN:
164 		wqe_size = WQE_LCMD_SIZE;
165 		break;
166 	case BUFDESC_SCMD_LEN:
167 		wqe_size = WQE_SCMD_SIZE;
168 		break;
169 	}
170 
171 	return wqe_size;
172 }
173 
cmdq_set_sge_completion(struct hinic_cmdq_completion * completion,struct hinic_cmdq_buf * buf_out)174 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion,
175 				    struct hinic_cmdq_buf *buf_out)
176 {
177 	struct hinic_sge_resp *sge_resp = &completion->sge_resp;
178 
179 	hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size);
180 }
181 
cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe * wqe,int wrapped,enum hinic_cmd_ack_type ack_type,enum hinic_mod_type mod,u8 cmd,u16 prod_idx,enum completion_format complete_format,enum data_format data_format,enum bufdesc_len buf_len)182 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
183 				  enum hinic_cmd_ack_type ack_type,
184 				  enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
185 				  enum completion_format complete_format,
186 				  enum data_format data_format,
187 				  enum bufdesc_len buf_len)
188 {
189 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
190 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
191 	enum ctrl_sect_len ctrl_len;
192 	struct hinic_ctrl *ctrl;
193 	u32 saved_data;
194 
195 	if (data_format == DATA_SGE) {
196 		wqe_lcmd = &wqe->wqe_lcmd;
197 
198 		wqe_lcmd->status.status_info = 0;
199 		ctrl = &wqe_lcmd->ctrl;
200 		ctrl_len = CTRL_SECT_LEN;
201 	} else {
202 		wqe_scmd = &wqe->direct_wqe.wqe_scmd;
203 
204 		wqe_scmd->status.status_info = 0;
205 		ctrl = &wqe_scmd->ctrl;
206 		ctrl_len = CTRL_DIRECT_SECT_LEN;
207 	}
208 
209 	ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI)             |
210 			  HINIC_CMDQ_CTRL_SET(cmd, CMD)                 |
211 			  HINIC_CMDQ_CTRL_SET(mod, MOD)                 |
212 			  HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE);
213 
214 	CMDQ_WQE_HEADER(wqe)->header_info =
215 		HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN)            |
216 		HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT)   |
217 		HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT)           |
218 		HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)           |
219 		HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
220 		HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)              |
221 		HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED);
222 
223 	saved_data = CMDQ_WQE_HEADER(wqe)->saved_data;
224 	saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM);
225 
226 	if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM)
227 		CMDQ_WQE_HEADER(wqe)->saved_data |=
228 						HINIC_SAVED_DATA_SET(1, ARM);
229 	else
230 		CMDQ_WQE_HEADER(wqe)->saved_data = saved_data;
231 }
232 
cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd * wqe_lcmd,struct hinic_cmdq_buf * buf_in)233 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd,
234 				  struct hinic_cmdq_buf *buf_in)
235 {
236 	hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size);
237 }
238 
cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe * wqe,void * buf_in,u32 in_size)239 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe,
240 				     void *buf_in, u32 in_size)
241 {
242 	struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd;
243 
244 	wqe_scmd->buf_desc.buf_len = in_size;
245 	memcpy(wqe_scmd->buf_desc.data, buf_in, in_size);
246 }
247 
cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe * wqe,enum cmdq_cmd_type cmd_type,struct hinic_cmdq_buf * buf_in,struct hinic_cmdq_buf * buf_out,int wrapped,enum hinic_cmd_ack_type ack_type,enum hinic_mod_type mod,u8 cmd,u16 prod_idx)248 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
249 			      enum cmdq_cmd_type cmd_type,
250 			      struct hinic_cmdq_buf *buf_in,
251 			      struct hinic_cmdq_buf *buf_out, int wrapped,
252 			      enum hinic_cmd_ack_type ack_type,
253 			      enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
254 {
255 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
256 	enum completion_format complete_format;
257 
258 	switch (cmd_type) {
259 	case CMDQ_CMD_SYNC_SGE_RESP:
260 		complete_format = COMPLETE_SGE;
261 		cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out);
262 		break;
263 	case CMDQ_CMD_SYNC_DIRECT_RESP:
264 		complete_format = COMPLETE_DIRECT;
265 		wqe_lcmd->completion.direct_resp = 0;
266 		break;
267 	}
268 
269 	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
270 			      prod_idx, complete_format, DATA_SGE,
271 			      BUFDESC_LCMD_LEN);
272 
273 	cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
274 }
275 
cmdq_set_direct_wqe(struct hinic_cmdq_wqe * wqe,enum cmdq_cmd_type cmd_type,void * buf_in,u16 in_size,struct hinic_cmdq_buf * buf_out,int wrapped,enum hinic_cmd_ack_type ack_type,enum hinic_mod_type mod,u8 cmd,u16 prod_idx)276 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe,
277 				enum cmdq_cmd_type cmd_type,
278 				void *buf_in, u16 in_size,
279 				struct hinic_cmdq_buf *buf_out, int wrapped,
280 				enum hinic_cmd_ack_type ack_type,
281 				enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
282 {
283 	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
284 	enum completion_format complete_format;
285 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
286 
287 	wqe_scmd = &direct_wqe->wqe_scmd;
288 
289 	switch (cmd_type) {
290 	case CMDQ_CMD_SYNC_SGE_RESP:
291 		complete_format = COMPLETE_SGE;
292 		cmdq_set_sge_completion(&wqe_scmd->completion, buf_out);
293 		break;
294 	case CMDQ_CMD_SYNC_DIRECT_RESP:
295 		complete_format = COMPLETE_DIRECT;
296 		wqe_scmd->completion.direct_resp = 0;
297 		break;
298 	}
299 
300 	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx,
301 			      complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN);
302 
303 	cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size);
304 }
305 
cmdq_wqe_fill(void * dst,void * src)306 static void cmdq_wqe_fill(void *dst, void *src)
307 {
308 	memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST,
309 	       CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
310 
311 	wmb();          /* The first 8 bytes should be written last */
312 
313 	*(u64 *)dst = *(u64 *)src;
314 }
315 
cmdq_fill_db(u32 * db_info,enum hinic_cmdq_type cmdq_type,u16 prod_idx)316 static void cmdq_fill_db(u32 *db_info,
317 			 enum hinic_cmdq_type cmdq_type, u16 prod_idx)
318 {
319 	*db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
320 		   HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH)               |
321 		   HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)                |
322 		   HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE);
323 }
324 
cmdq_set_db(struct hinic_cmdq * cmdq,enum hinic_cmdq_type cmdq_type,u16 prod_idx)325 static void cmdq_set_db(struct hinic_cmdq *cmdq,
326 			enum hinic_cmdq_type cmdq_type, u16 prod_idx)
327 {
328 	u32 db_info;
329 
330 	cmdq_fill_db(&db_info, cmdq_type, prod_idx);
331 
332 	/* The data that is written to HW should be in Big Endian Format */
333 	db_info = cpu_to_be32(db_info);
334 
335 	wmb();  /* write all before the doorbell */
336 
337 	writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
338 }
339 
cmdq_sync_cmd_direct_resp(struct hinic_cmdq * cmdq,enum hinic_mod_type mod,u8 cmd,struct hinic_cmdq_buf * buf_in,u64 * resp)340 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
341 				     enum hinic_mod_type mod, u8 cmd,
342 				     struct hinic_cmdq_buf *buf_in,
343 				     u64 *resp)
344 {
345 	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
346 	u16 curr_prod_idx, next_prod_idx;
347 	int errcode, wrapped, num_wqebbs;
348 	struct hinic_wq *wq = cmdq->wq;
349 	struct hinic_hw_wqe *hw_wqe;
350 	struct completion done;
351 
352 	/* Keep doorbell index correct. bh - for tasklet(ceq). */
353 	spin_lock_bh(&cmdq->cmdq_lock);
354 
355 	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
356 	hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx);
357 	if (IS_ERR(hw_wqe)) {
358 		spin_unlock_bh(&cmdq->cmdq_lock);
359 		return -EBUSY;
360 	}
361 
362 	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
363 
364 	wrapped = cmdq->wrapped;
365 
366 	num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
367 	next_prod_idx = curr_prod_idx + num_wqebbs;
368 	if (next_prod_idx >= wq->q_depth) {
369 		cmdq->wrapped = !cmdq->wrapped;
370 		next_prod_idx -= wq->q_depth;
371 	}
372 
373 	cmdq->errcode[curr_prod_idx] = &errcode;
374 
375 	init_completion(&done);
376 	cmdq->done[curr_prod_idx] = &done;
377 
378 	cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL,
379 			  wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd,
380 			  curr_prod_idx);
381 
382 	/* The data that is written to HW should be in Big Endian Format */
383 	hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE);
384 
385 	/* CMDQ WQE is not shadow, therefore wqe will be written to wq */
386 	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
387 
388 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
389 
390 	spin_unlock_bh(&cmdq->cmdq_lock);
391 
392 	if (!wait_for_completion_timeout(&done,
393 					 msecs_to_jiffies(CMDQ_TIMEOUT))) {
394 		spin_lock_bh(&cmdq->cmdq_lock);
395 
396 		if (cmdq->errcode[curr_prod_idx] == &errcode)
397 			cmdq->errcode[curr_prod_idx] = NULL;
398 
399 		if (cmdq->done[curr_prod_idx] == &done)
400 			cmdq->done[curr_prod_idx] = NULL;
401 
402 		spin_unlock_bh(&cmdq->cmdq_lock);
403 
404 		hinic_dump_ceq_info(cmdq->hwdev);
405 		return -ETIMEDOUT;
406 	}
407 
408 	smp_rmb();      /* read error code after completion */
409 
410 	if (resp) {
411 		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd;
412 
413 		*resp = cpu_to_be64(wqe_lcmd->completion.direct_resp);
414 	}
415 
416 	if (errcode != 0)
417 		return -EFAULT;
418 
419 	return 0;
420 }
421 
cmdq_set_arm_bit(struct hinic_cmdq * cmdq,void * buf_in,u16 in_size)422 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in,
423 			    u16 in_size)
424 {
425 	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
426 	u16 curr_prod_idx, next_prod_idx;
427 	struct hinic_wq *wq = cmdq->wq;
428 	struct hinic_hw_wqe *hw_wqe;
429 	int wrapped, num_wqebbs;
430 
431 	/* Keep doorbell index correct */
432 	spin_lock(&cmdq->cmdq_lock);
433 
434 	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
435 	hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx);
436 	if (IS_ERR(hw_wqe)) {
437 		spin_unlock(&cmdq->cmdq_lock);
438 		return -EBUSY;
439 	}
440 
441 	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
442 
443 	wrapped = cmdq->wrapped;
444 
445 	num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
446 	next_prod_idx = curr_prod_idx + num_wqebbs;
447 	if (next_prod_idx >= wq->q_depth) {
448 		cmdq->wrapped = !cmdq->wrapped;
449 		next_prod_idx -= wq->q_depth;
450 	}
451 
452 	cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in,
453 			    in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ,
454 			    HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx);
455 
456 	/* The data that is written to HW should be in Big Endian Format */
457 	hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE);
458 
459 	/* cmdq wqe is not shadow, therefore wqe will be written to wq */
460 	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
461 
462 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
463 
464 	spin_unlock(&cmdq->cmdq_lock);
465 	return 0;
466 }
467 
cmdq_params_valid(struct hinic_cmdq_buf * buf_in)468 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in)
469 {
470 	if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE)
471 		return -EINVAL;
472 
473 	return 0;
474 }
475 
476 /**
477  * hinic_cmdq_direct_resp - send command with direct data as resp
478  * @cmdqs: the cmdqs
479  * @mod: module on the card that will handle the command
480  * @cmd: the command
481  * @buf_in: the buffer for the command
482  * @resp: the response to return
483  *
484  * Return 0 - Success, negative - Failure
485  **/
hinic_cmdq_direct_resp(struct hinic_cmdqs * cmdqs,enum hinic_mod_type mod,u8 cmd,struct hinic_cmdq_buf * buf_in,u64 * resp)486 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
487 			   enum hinic_mod_type mod, u8 cmd,
488 			   struct hinic_cmdq_buf *buf_in, u64 *resp)
489 {
490 	struct hinic_hwif *hwif = cmdqs->hwif;
491 	struct pci_dev *pdev = hwif->pdev;
492 	int err;
493 
494 	err = cmdq_params_valid(buf_in);
495 	if (err) {
496 		dev_err(&pdev->dev, "Invalid CMDQ parameters\n");
497 		return err;
498 	}
499 
500 	return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
501 					 mod, cmd, buf_in, resp);
502 }
503 
504 /**
505  * hinic_set_arm_bit - set arm bit for enable interrupt again
506  * @cmdqs: the cmdqs
507  * @q_type: type of queue to set the arm bit for
508  * @q_id: the queue number
509  *
510  * Return 0 - Success, negative - Failure
511  **/
hinic_set_arm_bit(struct hinic_cmdqs * cmdqs,enum hinic_set_arm_qtype q_type,u32 q_id)512 int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
513 		      enum hinic_set_arm_qtype q_type, u32 q_id)
514 {
515 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];
516 	struct hinic_hwif *hwif = cmdqs->hwif;
517 	struct pci_dev *pdev = hwif->pdev;
518 	struct hinic_cmdq_arm_bit arm_bit;
519 	int err;
520 
521 	arm_bit.q_type = q_type;
522 	arm_bit.q_id   = q_id;
523 
524 	err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit));
525 	if (err) {
526 		dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id);
527 		return err;
528 	}
529 
530 	return 0;
531 }
532 
clear_wqe_complete_bit(struct hinic_cmdq * cmdq,struct hinic_cmdq_wqe * wqe)533 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
534 				   struct hinic_cmdq_wqe *wqe)
535 {
536 	u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info);
537 	unsigned int bufdesc_len, wqe_size;
538 	struct hinic_ctrl *ctrl;
539 
540 	bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
541 	wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len);
542 	if (wqe_size == WQE_LCMD_SIZE) {
543 		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
544 
545 		ctrl = &wqe_lcmd->ctrl;
546 	} else {
547 		struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
548 		struct hinic_cmdq_wqe_scmd *wqe_scmd;
549 
550 		wqe_scmd = &direct_wqe->wqe_scmd;
551 		ctrl = &wqe_scmd->ctrl;
552 	}
553 
554 	/* clear HW busy bit */
555 	ctrl->ctrl_info = 0;
556 
557 	wmb();  /* verify wqe is clear */
558 }
559 
560 /**
561  * cmdq_arm_ceq_handler - cmdq completion event handler for arm command
562  * @cmdq: the cmdq of the arm command
563  * @wqe: the wqe of the arm command
564  *
565  * Return 0 - Success, negative - Failure
566  **/
cmdq_arm_ceq_handler(struct hinic_cmdq * cmdq,struct hinic_cmdq_wqe * wqe)567 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq,
568 				struct hinic_cmdq_wqe *wqe)
569 {
570 	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
571 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
572 	struct hinic_ctrl *ctrl;
573 	u32 ctrl_info;
574 
575 	wqe_scmd = &direct_wqe->wqe_scmd;
576 	ctrl = &wqe_scmd->ctrl;
577 	ctrl_info = be32_to_cpu(ctrl->ctrl_info);
578 
579 	/* HW should toggle the HW BUSY BIT */
580 	if (!CMDQ_WQE_COMPLETED(ctrl_info))
581 		return -EBUSY;
582 
583 	clear_wqe_complete_bit(cmdq, wqe);
584 
585 	hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE);
586 	return 0;
587 }
588 
cmdq_update_errcode(struct hinic_cmdq * cmdq,u16 prod_idx,int errcode)589 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
590 				int errcode)
591 {
592 	if (cmdq->errcode[prod_idx])
593 		*cmdq->errcode[prod_idx] = errcode;
594 }
595 
596 /**
597  * cmdq_sync_cmd_handler - cmdq completion event handler for sync command
598  * @cmdq: the cmdq of the command
599  * @cons_idx: the consumer index to update the error code for
600  * @errcode: the error code
601  **/
cmdq_sync_cmd_handler(struct hinic_cmdq * cmdq,u16 cons_idx,int errcode)602 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx,
603 				  int errcode)
604 {
605 	u16 prod_idx = cons_idx;
606 
607 	spin_lock(&cmdq->cmdq_lock);
608 	cmdq_update_errcode(cmdq, prod_idx, errcode);
609 
610 	wmb();  /* write all before update for the command request */
611 
612 	if (cmdq->done[prod_idx])
613 		complete(cmdq->done[prod_idx]);
614 	spin_unlock(&cmdq->cmdq_lock);
615 }
616 
cmdq_cmd_ceq_handler(struct hinic_cmdq * cmdq,u16 ci,struct hinic_cmdq_wqe * cmdq_wqe)617 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci,
618 				struct hinic_cmdq_wqe *cmdq_wqe)
619 {
620 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd;
621 	struct hinic_status *status = &wqe_lcmd->status;
622 	struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl;
623 	int errcode;
624 
625 	if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info)))
626 		return -EBUSY;
627 
628 	dma_rmb();
629 
630 	errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL);
631 
632 	cmdq_sync_cmd_handler(cmdq, ci, errcode);
633 
634 	clear_wqe_complete_bit(cmdq, cmdq_wqe);
635 	hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE);
636 	return 0;
637 }
638 
639 /**
640  * cmdq_ceq_handler - cmdq completion event handler
641  * @handle: private data for the handler(cmdqs)
642  * @ceqe_data: ceq element data
643  **/
cmdq_ceq_handler(void * handle,u32 ceqe_data)644 static void cmdq_ceq_handler(void *handle, u32 ceqe_data)
645 {
646 	enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE);
647 	struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle;
648 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type];
649 	struct hinic_cmdq_header *header;
650 	struct hinic_hw_wqe *hw_wqe;
651 	int err, set_arm = 0;
652 	u32 saved_data;
653 	u16 ci;
654 
655 	/* Read the smallest wqe size for getting wqe size */
656 	while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) {
657 		if (IS_ERR(hw_wqe))
658 			break;
659 
660 		header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe);
661 		saved_data = be32_to_cpu(header->saved_data);
662 
663 		if (HINIC_SAVED_DATA_GET(saved_data, ARM)) {
664 			/* arm_bit was set until here */
665 			set_arm = 0;
666 
667 			if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe))
668 				break;
669 		} else {
670 			set_arm = 1;
671 
672 			hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci);
673 			if (IS_ERR(hw_wqe))
674 				break;
675 
676 			if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe))
677 				break;
678 		}
679 	}
680 
681 	if (set_arm) {
682 		struct hinic_hwif *hwif = cmdqs->hwif;
683 		struct pci_dev *pdev = hwif->pdev;
684 
685 		err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type);
686 		if (err)
687 			dev_err(&pdev->dev, "Failed to set arm for CMDQ\n");
688 	}
689 }
690 
691 /**
692  * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq
693  * @cmdq_ctxt: cmdq ctxt to initialize
694  * @cmdq: the cmdq
695  * @cmdq_pages: the memory of the queue
696  **/
cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt * cmdq_ctxt,struct hinic_cmdq * cmdq,struct hinic_cmdq_pages * cmdq_pages)697 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt,
698 				 struct hinic_cmdq *cmdq,
699 				 struct hinic_cmdq_pages *cmdq_pages)
700 {
701 	struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
702 	u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn;
703 	struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq);
704 	struct hinic_wq *wq = cmdq->wq;
705 
706 	/* The data in the HW is in Big Endian Format */
707 	wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr);
708 
709 	pfn = CMDQ_PFN(wq_first_page_paddr, SZ_4K);
710 
711 	ctxt_info->curr_wqe_page_pfn =
712 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN)   |
713 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
714 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM)               |
715 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)                |
716 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED);
717 
718 	if (wq->num_q_pages != 1) {
719 		/* block PFN - Read Modify Write */
720 		cmdq_first_block_paddr = cmdq_pages->page_paddr;
721 
722 		pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size);
723 	}
724 
725 	ctxt_info->wq_block_pfn =
726 		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) |
727 		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI);
728 
729 	cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif);
730 	cmdq_ctxt->ppf_idx = HINIC_HWIF_PPF_IDX(cmdqs->hwif);
731 	cmdq_ctxt->cmdq_type  = cmdq->cmdq_type;
732 }
733 
734 /**
735  * init_cmdq - initialize cmdq
736  * @cmdq: the cmdq
737  * @wq: the wq attaced to the cmdq
738  * @q_type: the cmdq type of the cmdq
739  * @db_area: doorbell area for the cmdq
740  *
741  * Return 0 - Success, negative - Failure
742  **/
init_cmdq(struct hinic_cmdq * cmdq,struct hinic_wq * wq,enum hinic_cmdq_type q_type,void __iomem * db_area)743 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq,
744 		     enum hinic_cmdq_type q_type, void __iomem *db_area)
745 {
746 	int err;
747 
748 	cmdq->wq = wq;
749 	cmdq->cmdq_type = q_type;
750 	cmdq->wrapped = 1;
751 
752 	spin_lock_init(&cmdq->cmdq_lock);
753 
754 	cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth));
755 	if (!cmdq->done)
756 		return -ENOMEM;
757 
758 	cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode),
759 					   wq->q_depth));
760 	if (!cmdq->errcode) {
761 		err = -ENOMEM;
762 		goto err_errcode;
763 	}
764 
765 	cmdq->db_base = db_area + CMDQ_DB_OFF;
766 	return 0;
767 
768 err_errcode:
769 	vfree(cmdq->done);
770 	return err;
771 }
772 
773 /**
774  * free_cmdq - Free cmdq
775  * @cmdq: the cmdq to free
776  **/
free_cmdq(struct hinic_cmdq * cmdq)777 static void free_cmdq(struct hinic_cmdq *cmdq)
778 {
779 	vfree(cmdq->errcode);
780 	vfree(cmdq->done);
781 }
782 
783 /**
784  * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq
785  * @hwdev: the NIC HW device
786  * @cmdqs: cmdqs to write the ctxts for
787  * @db_area: db_area for all the cmdqs
788  *
789  * Return 0 - Success, negative - Failure
790  **/
init_cmdqs_ctxt(struct hinic_hwdev * hwdev,struct hinic_cmdqs * cmdqs,void __iomem ** db_area)791 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev,
792 			   struct hinic_cmdqs *cmdqs, void __iomem **db_area)
793 {
794 	struct hinic_hwif *hwif = hwdev->hwif;
795 	enum hinic_cmdq_type type, cmdq_type;
796 	struct hinic_cmdq_ctxt *cmdq_ctxts;
797 	struct pci_dev *pdev = hwif->pdev;
798 	struct hinic_pfhwdev *pfhwdev;
799 	int err;
800 
801 	cmdq_ctxts = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES,
802 				  sizeof(*cmdq_ctxts), GFP_KERNEL);
803 	if (!cmdq_ctxts)
804 		return -ENOMEM;
805 
806 	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
807 
808 	cmdq_type = HINIC_CMDQ_SYNC;
809 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
810 		cmdqs->cmdq[cmdq_type].hwdev = hwdev;
811 		err = init_cmdq(&cmdqs->cmdq[cmdq_type],
812 				&cmdqs->saved_wqs[cmdq_type], cmdq_type,
813 				db_area[cmdq_type]);
814 		if (err) {
815 			dev_err(&pdev->dev, "Failed to initialize cmdq\n");
816 			goto err_init_cmdq;
817 		}
818 
819 		cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type],
820 				     &cmdqs->cmdq[cmdq_type],
821 				     &cmdqs->cmdq_pages);
822 	}
823 
824 	/* Write the CMDQ ctxts */
825 	cmdq_type = HINIC_CMDQ_SYNC;
826 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
827 		err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
828 					HINIC_COMM_CMD_CMDQ_CTXT_SET,
829 					&cmdq_ctxts[cmdq_type],
830 					sizeof(cmdq_ctxts[cmdq_type]),
831 					NULL, NULL, HINIC_MGMT_MSG_SYNC);
832 		if (err) {
833 			dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n",
834 				cmdq_type);
835 			goto err_write_cmdq_ctxt;
836 		}
837 	}
838 
839 	devm_kfree(&pdev->dev, cmdq_ctxts);
840 	return 0;
841 
842 err_write_cmdq_ctxt:
843 	cmdq_type = HINIC_MAX_CMDQ_TYPES;
844 
845 err_init_cmdq:
846 	for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++)
847 		free_cmdq(&cmdqs->cmdq[type]);
848 
849 	devm_kfree(&pdev->dev, cmdq_ctxts);
850 	return err;
851 }
852 
hinic_set_cmdq_depth(struct hinic_hwdev * hwdev,u16 cmdq_depth)853 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth)
854 {
855 	struct hinic_cmd_hw_ioctxt hw_ioctxt = { 0 };
856 	struct hinic_pfhwdev *pfhwdev;
857 
858 	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
859 
860 	hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
861 	hw_ioctxt.ppf_idx = HINIC_HWIF_PPF_IDX(hwdev->hwif);
862 
863 	hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_ENABLE;
864 	hw_ioctxt.cmdq_depth = (u8)ilog2(cmdq_depth);
865 
866 	return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
867 				 HINIC_COMM_CMD_HWCTXT_SET,
868 				 &hw_ioctxt, sizeof(hw_ioctxt), NULL,
869 				 NULL, HINIC_MGMT_MSG_SYNC);
870 }
871 
872 /**
873  * hinic_init_cmdqs - init all cmdqs
874  * @cmdqs: cmdqs to init
875  * @hwif: HW interface for accessing cmdqs
876  * @db_area: doorbell areas for all the cmdqs
877  *
878  * Return 0 - Success, negative - Failure
879  **/
hinic_init_cmdqs(struct hinic_cmdqs * cmdqs,struct hinic_hwif * hwif,void __iomem ** db_area)880 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
881 		     void __iomem **db_area)
882 {
883 	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
884 	struct pci_dev *pdev = hwif->pdev;
885 	struct hinic_hwdev *hwdev;
886 	u16 max_wqe_size;
887 	int err;
888 
889 	cmdqs->hwif = hwif;
890 	cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev,
891 					       HINIC_CMDQ_BUF_SIZE,
892 					       HINIC_CMDQ_BUF_SIZE, 0);
893 	if (!cmdqs->cmdq_buf_pool)
894 		return -ENOMEM;
895 
896 	cmdqs->saved_wqs = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES,
897 					sizeof(*cmdqs->saved_wqs), GFP_KERNEL);
898 	if (!cmdqs->saved_wqs) {
899 		err = -ENOMEM;
900 		goto err_saved_wqs;
901 	}
902 
903 	max_wqe_size = WQE_LCMD_SIZE;
904 	err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif,
905 				   HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE,
906 				   CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size);
907 	if (err) {
908 		dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n");
909 		goto err_cmdq_wqs;
910 	}
911 
912 	hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io);
913 	err = init_cmdqs_ctxt(hwdev, cmdqs, db_area);
914 	if (err) {
915 		dev_err(&pdev->dev, "Failed to write cmdq ctxt\n");
916 		goto err_cmdq_ctxt;
917 	}
918 
919 	hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs,
920 			      cmdq_ceq_handler);
921 
922 	err = hinic_set_cmdq_depth(hwdev, CMDQ_DEPTH);
923 	if (err) {
924 		dev_err(&hwif->pdev->dev, "Failed to set cmdq depth\n");
925 		goto err_set_cmdq_depth;
926 	}
927 
928 	return 0;
929 
930 err_set_cmdq_depth:
931 	hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ);
932 
933 err_cmdq_ctxt:
934 	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
935 			    HINIC_MAX_CMDQ_TYPES);
936 
937 err_cmdq_wqs:
938 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
939 
940 err_saved_wqs:
941 	dma_pool_destroy(cmdqs->cmdq_buf_pool);
942 	return err;
943 }
944 
945 /**
946  * hinic_free_cmdqs - free all cmdqs
947  * @cmdqs: cmdqs to free
948  **/
hinic_free_cmdqs(struct hinic_cmdqs * cmdqs)949 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs)
950 {
951 	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
952 	struct hinic_hwif *hwif = cmdqs->hwif;
953 	struct pci_dev *pdev = hwif->pdev;
954 	enum hinic_cmdq_type cmdq_type;
955 
956 	hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ);
957 
958 	cmdq_type = HINIC_CMDQ_SYNC;
959 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
960 		free_cmdq(&cmdqs->cmdq[cmdq_type]);
961 
962 	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
963 			    HINIC_MAX_CMDQ_TYPES);
964 
965 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
966 
967 	dma_pool_destroy(cmdqs->cmdq_buf_pool);
968 }
969