1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016 Cavium, Inc.
4  */
5 
6 #include <linux/interrupt.h>
7 #include <linux/module.h>
8 
9 #include "cptvf.h"
10 
11 #define DRV_NAME	"thunder-cptvf"
12 #define DRV_VERSION	"1.0"
13 
14 struct cptvf_wqe {
15 	struct tasklet_struct twork;
16 	void *cptvf;
17 	u32 qno;
18 };
19 
20 struct cptvf_wqe_info {
21 	struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
22 };
23 
vq_work_handler(unsigned long data)24 static void vq_work_handler(unsigned long data)
25 {
26 	struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
27 	struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
28 
29 	vq_post_process(cwqe->cptvf, cwqe->qno);
30 }
31 
init_worker_threads(struct cpt_vf * cptvf)32 static int init_worker_threads(struct cpt_vf *cptvf)
33 {
34 	struct pci_dev *pdev = cptvf->pdev;
35 	struct cptvf_wqe_info *cwqe_info;
36 	int i;
37 
38 	cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
39 	if (!cwqe_info)
40 		return -ENOMEM;
41 
42 	if (cptvf->nr_queues) {
43 		dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
44 			 cptvf->nr_queues);
45 	}
46 
47 	for (i = 0; i < cptvf->nr_queues; i++) {
48 		tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
49 			     (u64)cwqe_info);
50 		cwqe_info->vq_wqe[i].qno = i;
51 		cwqe_info->vq_wqe[i].cptvf = cptvf;
52 	}
53 
54 	cptvf->wqe_info = cwqe_info;
55 
56 	return 0;
57 }
58 
cleanup_worker_threads(struct cpt_vf * cptvf)59 static void cleanup_worker_threads(struct cpt_vf *cptvf)
60 {
61 	struct cptvf_wqe_info *cwqe_info;
62 	struct pci_dev *pdev = cptvf->pdev;
63 	int i;
64 
65 	cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
66 	if (!cwqe_info)
67 		return;
68 
69 	if (cptvf->nr_queues) {
70 		dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
71 			 cptvf->nr_queues);
72 	}
73 
74 	for (i = 0; i < cptvf->nr_queues; i++)
75 		tasklet_kill(&cwqe_info->vq_wqe[i].twork);
76 
77 	kfree_sensitive(cwqe_info);
78 	cptvf->wqe_info = NULL;
79 }
80 
free_pending_queues(struct pending_qinfo * pqinfo)81 static void free_pending_queues(struct pending_qinfo *pqinfo)
82 {
83 	int i;
84 	struct pending_queue *queue;
85 
86 	for_each_pending_queue(pqinfo, queue, i) {
87 		if (!queue->head)
88 			continue;
89 
90 		/* free single queue */
91 		kfree_sensitive((queue->head));
92 
93 		queue->front = 0;
94 		queue->rear = 0;
95 
96 		return;
97 	}
98 
99 	pqinfo->qlen = 0;
100 	pqinfo->nr_queues = 0;
101 }
102 
alloc_pending_queues(struct pending_qinfo * pqinfo,u32 qlen,u32 nr_queues)103 static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
104 				u32 nr_queues)
105 {
106 	u32 i;
107 	int ret;
108 	struct pending_queue *queue = NULL;
109 
110 	pqinfo->nr_queues = nr_queues;
111 	pqinfo->qlen = qlen;
112 
113 	for_each_pending_queue(pqinfo, queue, i) {
114 		queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
115 		if (!queue->head) {
116 			ret = -ENOMEM;
117 			goto pending_qfail;
118 		}
119 
120 		queue->front = 0;
121 		queue->rear = 0;
122 		atomic64_set((&queue->pending_count), (0));
123 
124 		/* init queue spin lock */
125 		spin_lock_init(&queue->lock);
126 	}
127 
128 	return 0;
129 
130 pending_qfail:
131 	free_pending_queues(pqinfo);
132 
133 	return ret;
134 }
135 
init_pending_queues(struct cpt_vf * cptvf,u32 qlen,u32 nr_queues)136 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
137 {
138 	struct pci_dev *pdev = cptvf->pdev;
139 	int ret;
140 
141 	if (!nr_queues)
142 		return 0;
143 
144 	ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
145 	if (ret) {
146 		dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
147 			nr_queues);
148 		return ret;
149 	}
150 
151 	return 0;
152 }
153 
cleanup_pending_queues(struct cpt_vf * cptvf)154 static void cleanup_pending_queues(struct cpt_vf *cptvf)
155 {
156 	struct pci_dev *pdev = cptvf->pdev;
157 
158 	if (!cptvf->nr_queues)
159 		return;
160 
161 	dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
162 		 cptvf->nr_queues);
163 	free_pending_queues(&cptvf->pqinfo);
164 }
165 
free_command_queues(struct cpt_vf * cptvf,struct command_qinfo * cqinfo)166 static void free_command_queues(struct cpt_vf *cptvf,
167 				struct command_qinfo *cqinfo)
168 {
169 	int i;
170 	struct command_queue *queue = NULL;
171 	struct command_chunk *chunk = NULL;
172 	struct pci_dev *pdev = cptvf->pdev;
173 	struct hlist_node *node;
174 
175 	/* clean up for each queue */
176 	for (i = 0; i < cptvf->nr_queues; i++) {
177 		queue = &cqinfo->queue[i];
178 		if (hlist_empty(&cqinfo->queue[i].chead))
179 			continue;
180 
181 		hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
182 					  nextchunk) {
183 			dma_free_coherent(&pdev->dev, chunk->size,
184 					  chunk->head,
185 					  chunk->dma_addr);
186 			chunk->head = NULL;
187 			chunk->dma_addr = 0;
188 			hlist_del(&chunk->nextchunk);
189 			kfree_sensitive(chunk);
190 		}
191 
192 		queue->nchunks = 0;
193 		queue->idx = 0;
194 	}
195 
196 	/* common cleanup */
197 	cqinfo->cmd_size = 0;
198 }
199 
alloc_command_queues(struct cpt_vf * cptvf,struct command_qinfo * cqinfo,size_t cmd_size,u32 qlen)200 static int alloc_command_queues(struct cpt_vf *cptvf,
201 				struct command_qinfo *cqinfo, size_t cmd_size,
202 				u32 qlen)
203 {
204 	int i;
205 	size_t q_size;
206 	struct command_queue *queue = NULL;
207 	struct pci_dev *pdev = cptvf->pdev;
208 
209 	/* common init */
210 	cqinfo->cmd_size = cmd_size;
211 	/* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
212 	cptvf->qsize = min(qlen, cqinfo->qchunksize) *
213 			CPT_NEXT_CHUNK_PTR_SIZE + 1;
214 	/* Qsize in bytes to create space for alignment */
215 	q_size = qlen * cqinfo->cmd_size;
216 
217 	/* per queue initialization */
218 	for (i = 0; i < cptvf->nr_queues; i++) {
219 		size_t c_size = 0;
220 		size_t rem_q_size = q_size;
221 		struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
222 		u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
223 
224 		queue = &cqinfo->queue[i];
225 		INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
226 		do {
227 			curr = kzalloc(sizeof(*curr), GFP_KERNEL);
228 			if (!curr)
229 				goto cmd_qfail;
230 
231 			c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
232 					rem_q_size;
233 			curr->head = dma_alloc_coherent(&pdev->dev,
234 							c_size + CPT_NEXT_CHUNK_PTR_SIZE,
235 							&curr->dma_addr,
236 							GFP_KERNEL);
237 			if (!curr->head) {
238 				dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
239 					i, queue->nchunks);
240 				kfree(curr);
241 				goto cmd_qfail;
242 			}
243 
244 			curr->size = c_size;
245 			if (queue->nchunks == 0) {
246 				hlist_add_head(&curr->nextchunk,
247 					       &cqinfo->queue[i].chead);
248 				first = curr;
249 			} else {
250 				hlist_add_behind(&curr->nextchunk,
251 						 &last->nextchunk);
252 			}
253 
254 			queue->nchunks++;
255 			rem_q_size -= c_size;
256 			if (last)
257 				*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
258 
259 			last = curr;
260 		} while (rem_q_size);
261 
262 		/* Make the queue circular */
263 		/* Tie back last chunk entry to head */
264 		curr = first;
265 		*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
266 		queue->qhead = curr;
267 		spin_lock_init(&queue->lock);
268 	}
269 	return 0;
270 
271 cmd_qfail:
272 	free_command_queues(cptvf, cqinfo);
273 	return -ENOMEM;
274 }
275 
init_command_queues(struct cpt_vf * cptvf,u32 qlen)276 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
277 {
278 	struct pci_dev *pdev = cptvf->pdev;
279 	int ret;
280 
281 	/* setup AE command queues */
282 	ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
283 				   qlen);
284 	if (ret) {
285 		dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
286 			cptvf->nr_queues);
287 		return ret;
288 	}
289 
290 	return ret;
291 }
292 
cleanup_command_queues(struct cpt_vf * cptvf)293 static void cleanup_command_queues(struct cpt_vf *cptvf)
294 {
295 	struct pci_dev *pdev = cptvf->pdev;
296 
297 	if (!cptvf->nr_queues)
298 		return;
299 
300 	dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
301 		 cptvf->nr_queues);
302 	free_command_queues(cptvf, &cptvf->cqinfo);
303 }
304 
cptvf_sw_cleanup(struct cpt_vf * cptvf)305 static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
306 {
307 	cleanup_worker_threads(cptvf);
308 	cleanup_pending_queues(cptvf);
309 	cleanup_command_queues(cptvf);
310 }
311 
cptvf_sw_init(struct cpt_vf * cptvf,u32 qlen,u32 nr_queues)312 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
313 {
314 	struct pci_dev *pdev = cptvf->pdev;
315 	int ret = 0;
316 	u32 max_dev_queues = 0;
317 
318 	max_dev_queues = CPT_NUM_QS_PER_VF;
319 	/* possible cpus */
320 	nr_queues = min_t(u32, nr_queues, max_dev_queues);
321 	cptvf->nr_queues = nr_queues;
322 
323 	ret = init_command_queues(cptvf, qlen);
324 	if (ret) {
325 		dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
326 			nr_queues);
327 		return ret;
328 	}
329 
330 	ret = init_pending_queues(cptvf, qlen, nr_queues);
331 	if (ret) {
332 		dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
333 			nr_queues);
334 		goto setup_pqfail;
335 	}
336 
337 	/* Create worker threads for BH processing */
338 	ret = init_worker_threads(cptvf);
339 	if (ret) {
340 		dev_err(&pdev->dev, "Failed to setup worker threads\n");
341 		goto init_work_fail;
342 	}
343 
344 	return 0;
345 
346 init_work_fail:
347 	cleanup_worker_threads(cptvf);
348 	cleanup_pending_queues(cptvf);
349 
350 setup_pqfail:
351 	cleanup_command_queues(cptvf);
352 
353 	return ret;
354 }
355 
cptvf_free_irq_affinity(struct cpt_vf * cptvf,int vec)356 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
357 {
358 	irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
359 	free_cpumask_var(cptvf->affinity_mask[vec]);
360 }
361 
cptvf_write_vq_ctl(struct cpt_vf * cptvf,bool val)362 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
363 {
364 	union cptx_vqx_ctl vqx_ctl;
365 
366 	vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
367 	vqx_ctl.s.ena = val;
368 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
369 }
370 
cptvf_write_vq_doorbell(struct cpt_vf * cptvf,u32 val)371 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
372 {
373 	union cptx_vqx_doorbell vqx_dbell;
374 
375 	vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
376 				     CPTX_VQX_DOORBELL(0, 0));
377 	vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
378 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
379 			vqx_dbell.u);
380 }
381 
cptvf_write_vq_inprog(struct cpt_vf * cptvf,u8 val)382 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
383 {
384 	union cptx_vqx_inprog vqx_inprg;
385 
386 	vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
387 	vqx_inprg.s.inflight = val;
388 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
389 }
390 
cptvf_write_vq_done_numwait(struct cpt_vf * cptvf,u32 val)391 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
392 {
393 	union cptx_vqx_done_wait vqx_dwait;
394 
395 	vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
396 				     CPTX_VQX_DONE_WAIT(0, 0));
397 	vqx_dwait.s.num_wait = val;
398 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
399 			vqx_dwait.u);
400 }
401 
cptvf_write_vq_done_timewait(struct cpt_vf * cptvf,u16 time)402 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
403 {
404 	union cptx_vqx_done_wait vqx_dwait;
405 
406 	vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
407 				     CPTX_VQX_DONE_WAIT(0, 0));
408 	vqx_dwait.s.time_wait = time;
409 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
410 			vqx_dwait.u);
411 }
412 
cptvf_enable_swerr_interrupts(struct cpt_vf * cptvf)413 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
414 {
415 	union cptx_vqx_misc_ena_w1s vqx_misc_ena;
416 
417 	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
418 					CPTX_VQX_MISC_ENA_W1S(0, 0));
419 	/* Set mbox(0) interupts for the requested vf */
420 	vqx_misc_ena.s.swerr = 1;
421 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
422 			vqx_misc_ena.u);
423 }
424 
cptvf_enable_mbox_interrupts(struct cpt_vf * cptvf)425 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
426 {
427 	union cptx_vqx_misc_ena_w1s vqx_misc_ena;
428 
429 	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
430 					CPTX_VQX_MISC_ENA_W1S(0, 0));
431 	/* Set mbox(0) interupts for the requested vf */
432 	vqx_misc_ena.s.mbox = 1;
433 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
434 			vqx_misc_ena.u);
435 }
436 
cptvf_enable_done_interrupts(struct cpt_vf * cptvf)437 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
438 {
439 	union cptx_vqx_done_ena_w1s vqx_done_ena;
440 
441 	vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
442 					CPTX_VQX_DONE_ENA_W1S(0, 0));
443 	/* Set DONE interrupt for the requested vf */
444 	vqx_done_ena.s.done = 1;
445 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
446 			vqx_done_ena.u);
447 }
448 
cptvf_clear_dovf_intr(struct cpt_vf * cptvf)449 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
450 {
451 	union cptx_vqx_misc_int vqx_misc_int;
452 
453 	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
454 					CPTX_VQX_MISC_INT(0, 0));
455 	/* W1C for the VF */
456 	vqx_misc_int.s.dovf = 1;
457 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
458 			vqx_misc_int.u);
459 }
460 
cptvf_clear_irde_intr(struct cpt_vf * cptvf)461 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
462 {
463 	union cptx_vqx_misc_int vqx_misc_int;
464 
465 	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
466 					CPTX_VQX_MISC_INT(0, 0));
467 	/* W1C for the VF */
468 	vqx_misc_int.s.irde = 1;
469 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
470 			vqx_misc_int.u);
471 }
472 
cptvf_clear_nwrp_intr(struct cpt_vf * cptvf)473 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
474 {
475 	union cptx_vqx_misc_int vqx_misc_int;
476 
477 	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
478 					CPTX_VQX_MISC_INT(0, 0));
479 	/* W1C for the VF */
480 	vqx_misc_int.s.nwrp = 1;
481 	cpt_write_csr64(cptvf->reg_base,
482 			CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
483 }
484 
cptvf_clear_mbox_intr(struct cpt_vf * cptvf)485 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
486 {
487 	union cptx_vqx_misc_int vqx_misc_int;
488 
489 	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
490 					CPTX_VQX_MISC_INT(0, 0));
491 	/* W1C for the VF */
492 	vqx_misc_int.s.mbox = 1;
493 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
494 			vqx_misc_int.u);
495 }
496 
cptvf_clear_swerr_intr(struct cpt_vf * cptvf)497 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
498 {
499 	union cptx_vqx_misc_int vqx_misc_int;
500 
501 	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
502 					CPTX_VQX_MISC_INT(0, 0));
503 	/* W1C for the VF */
504 	vqx_misc_int.s.swerr = 1;
505 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
506 			vqx_misc_int.u);
507 }
508 
cptvf_read_vf_misc_intr_status(struct cpt_vf * cptvf)509 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
510 {
511 	return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
512 }
513 
cptvf_misc_intr_handler(int irq,void * cptvf_irq)514 static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
515 {
516 	struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
517 	struct pci_dev *pdev = cptvf->pdev;
518 	u64 intr;
519 
520 	intr = cptvf_read_vf_misc_intr_status(cptvf);
521 	/*Check for MISC interrupt types*/
522 	if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
523 		dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
524 			intr, cptvf->vfid);
525 		cptvf_handle_mbox_intr(cptvf);
526 		cptvf_clear_mbox_intr(cptvf);
527 	} else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
528 		cptvf_clear_dovf_intr(cptvf);
529 		/*Clear doorbell count*/
530 		cptvf_write_vq_doorbell(cptvf, 0);
531 		dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
532 			intr, cptvf->vfid);
533 	} else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
534 		cptvf_clear_irde_intr(cptvf);
535 		dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
536 			intr, cptvf->vfid);
537 	} else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
538 		cptvf_clear_nwrp_intr(cptvf);
539 		dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
540 			intr, cptvf->vfid);
541 	} else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
542 		cptvf_clear_swerr_intr(cptvf);
543 		dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
544 			intr, cptvf->vfid);
545 	} else {
546 		dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
547 			cptvf->vfid);
548 	}
549 
550 	return IRQ_HANDLED;
551 }
552 
get_cptvf_vq_wqe(struct cpt_vf * cptvf,int qno)553 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
554 						 int qno)
555 {
556 	struct cptvf_wqe_info *nwqe_info;
557 
558 	if (unlikely(qno >= cptvf->nr_queues))
559 		return NULL;
560 	nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
561 
562 	return &nwqe_info->vq_wqe[qno];
563 }
564 
cptvf_read_vq_done_count(struct cpt_vf * cptvf)565 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
566 {
567 	union cptx_vqx_done vqx_done;
568 
569 	vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
570 	return vqx_done.s.done;
571 }
572 
cptvf_write_vq_done_ack(struct cpt_vf * cptvf,u32 ackcnt)573 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
574 					   u32 ackcnt)
575 {
576 	union cptx_vqx_done_ack vqx_dack_cnt;
577 
578 	vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
579 					CPTX_VQX_DONE_ACK(0, 0));
580 	vqx_dack_cnt.s.done_ack = ackcnt;
581 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
582 			vqx_dack_cnt.u);
583 }
584 
cptvf_done_intr_handler(int irq,void * cptvf_irq)585 static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
586 {
587 	struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
588 	struct pci_dev *pdev = cptvf->pdev;
589 	/* Read the number of completions */
590 	u32 intr = cptvf_read_vq_done_count(cptvf);
591 
592 	if (intr) {
593 		struct cptvf_wqe *wqe;
594 
595 		/* Acknowledge the number of
596 		 * scheduled completions for processing
597 		 */
598 		cptvf_write_vq_done_ack(cptvf, intr);
599 		wqe = get_cptvf_vq_wqe(cptvf, 0);
600 		if (unlikely(!wqe)) {
601 			dev_err(&pdev->dev, "No work to schedule for VF (%d)",
602 				cptvf->vfid);
603 			return IRQ_NONE;
604 		}
605 		tasklet_hi_schedule(&wqe->twork);
606 	}
607 
608 	return IRQ_HANDLED;
609 }
610 
cptvf_set_irq_affinity(struct cpt_vf * cptvf,int vec)611 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
612 {
613 	struct pci_dev *pdev = cptvf->pdev;
614 	int cpu;
615 
616 	if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
617 				GFP_KERNEL)) {
618 		dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
619 			cptvf->vfid);
620 		return;
621 	}
622 
623 	cpu = cptvf->vfid % num_online_cpus();
624 	cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
625 			cptvf->affinity_mask[vec]);
626 	irq_set_affinity_hint(pci_irq_vector(pdev, vec),
627 			cptvf->affinity_mask[vec]);
628 }
629 
cptvf_write_vq_saddr(struct cpt_vf * cptvf,u64 val)630 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
631 {
632 	union cptx_vqx_saddr vqx_saddr;
633 
634 	vqx_saddr.u = val;
635 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
636 }
637 
cptvf_device_init(struct cpt_vf * cptvf)638 static void cptvf_device_init(struct cpt_vf *cptvf)
639 {
640 	u64 base_addr = 0;
641 
642 	/* Disable the VQ */
643 	cptvf_write_vq_ctl(cptvf, 0);
644 	/* Reset the doorbell */
645 	cptvf_write_vq_doorbell(cptvf, 0);
646 	/* Clear inflight */
647 	cptvf_write_vq_inprog(cptvf, 0);
648 	/* Write VQ SADDR */
649 	/* TODO: for now only one queue, so hard coded */
650 	base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
651 	cptvf_write_vq_saddr(cptvf, base_addr);
652 	/* Configure timerhold / coalescence */
653 	cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
654 	cptvf_write_vq_done_numwait(cptvf, 1);
655 	/* Enable the VQ */
656 	cptvf_write_vq_ctl(cptvf, 1);
657 	/* Flag the VF ready */
658 	cptvf->flags |= CPT_FLAG_DEVICE_READY;
659 }
660 
cptvf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)661 static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
662 {
663 	struct device *dev = &pdev->dev;
664 	struct cpt_vf *cptvf;
665 	int    err;
666 
667 	cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
668 	if (!cptvf)
669 		return -ENOMEM;
670 
671 	pci_set_drvdata(pdev, cptvf);
672 	cptvf->pdev = pdev;
673 	err = pci_enable_device(pdev);
674 	if (err) {
675 		dev_err(dev, "Failed to enable PCI device\n");
676 		pci_set_drvdata(pdev, NULL);
677 		return err;
678 	}
679 
680 	err = pci_request_regions(pdev, DRV_NAME);
681 	if (err) {
682 		dev_err(dev, "PCI request regions failed 0x%x\n", err);
683 		goto cptvf_err_disable_device;
684 	}
685 	/* Mark as VF driver */
686 	cptvf->flags |= CPT_FLAG_VF_DRIVER;
687 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
688 	if (err) {
689 		dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
690 		goto cptvf_err_release_regions;
691 	}
692 
693 	/* MAP PF's configuration registers */
694 	cptvf->reg_base = pcim_iomap(pdev, 0, 0);
695 	if (!cptvf->reg_base) {
696 		dev_err(dev, "Cannot map config register space, aborting\n");
697 		err = -ENOMEM;
698 		goto cptvf_err_release_regions;
699 	}
700 
701 	cptvf->node = dev_to_node(&pdev->dev);
702 	err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
703 			CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
704 	if (err < 0) {
705 		dev_err(dev, "Request for #%d msix vectors failed\n",
706 			CPT_VF_MSIX_VECTORS);
707 		goto cptvf_err_release_regions;
708 	}
709 
710 	err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
711 			  cptvf_misc_intr_handler, 0, "CPT VF misc intr",
712 			  cptvf);
713 	if (err) {
714 		dev_err(dev, "Request misc irq failed");
715 		goto cptvf_free_vectors;
716 	}
717 
718 	/* Enable mailbox interrupt */
719 	cptvf_enable_mbox_interrupts(cptvf);
720 	cptvf_enable_swerr_interrupts(cptvf);
721 
722 	/* Check ready with PF */
723 	/* Gets chip ID / device Id from PF if ready */
724 	err = cptvf_check_pf_ready(cptvf);
725 	if (err) {
726 		dev_err(dev, "PF not responding to READY msg");
727 		goto cptvf_free_misc_irq;
728 	}
729 
730 	/* CPT VF software resources initialization */
731 	cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
732 	err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
733 	if (err) {
734 		dev_err(dev, "cptvf_sw_init() failed");
735 		goto cptvf_free_misc_irq;
736 	}
737 	/* Convey VQ LEN to PF */
738 	err = cptvf_send_vq_size_msg(cptvf);
739 	if (err) {
740 		dev_err(dev, "PF not responding to QLEN msg");
741 		goto cptvf_free_misc_irq;
742 	}
743 
744 	/* CPT VF device initialization */
745 	cptvf_device_init(cptvf);
746 	/* Send msg to PF to assign currnet Q to required group */
747 	cptvf->vfgrp = 1;
748 	err = cptvf_send_vf_to_grp_msg(cptvf);
749 	if (err) {
750 		dev_err(dev, "PF not responding to VF_GRP msg");
751 		goto cptvf_free_misc_irq;
752 	}
753 
754 	cptvf->priority = 1;
755 	err = cptvf_send_vf_priority_msg(cptvf);
756 	if (err) {
757 		dev_err(dev, "PF not responding to VF_PRIO msg");
758 		goto cptvf_free_misc_irq;
759 	}
760 
761 	err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
762 			  cptvf_done_intr_handler, 0, "CPT VF done intr",
763 			  cptvf);
764 	if (err) {
765 		dev_err(dev, "Request done irq failed\n");
766 		goto cptvf_free_misc_irq;
767 	}
768 
769 	/* Enable mailbox interrupt */
770 	cptvf_enable_done_interrupts(cptvf);
771 
772 	/* Set irq affinity masks */
773 	cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
774 	cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
775 
776 	err = cptvf_send_vf_up(cptvf);
777 	if (err) {
778 		dev_err(dev, "PF not responding to UP msg");
779 		goto cptvf_free_irq_affinity;
780 	}
781 	err = cvm_crypto_init(cptvf);
782 	if (err) {
783 		dev_err(dev, "Algorithm register failed\n");
784 		goto cptvf_free_irq_affinity;
785 	}
786 	return 0;
787 
788 cptvf_free_irq_affinity:
789 	cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
790 	cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
791 cptvf_free_misc_irq:
792 	free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
793 cptvf_free_vectors:
794 	pci_free_irq_vectors(cptvf->pdev);
795 cptvf_err_release_regions:
796 	pci_release_regions(pdev);
797 cptvf_err_disable_device:
798 	pci_disable_device(pdev);
799 	pci_set_drvdata(pdev, NULL);
800 
801 	return err;
802 }
803 
cptvf_remove(struct pci_dev * pdev)804 static void cptvf_remove(struct pci_dev *pdev)
805 {
806 	struct cpt_vf *cptvf = pci_get_drvdata(pdev);
807 
808 	if (!cptvf) {
809 		dev_err(&pdev->dev, "Invalid CPT-VF device\n");
810 		return;
811 	}
812 
813 	/* Convey DOWN to PF */
814 	if (cptvf_send_vf_down(cptvf)) {
815 		dev_err(&pdev->dev, "PF not responding to DOWN msg");
816 	} else {
817 		cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
818 		cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
819 		free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
820 		free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
821 		pci_free_irq_vectors(cptvf->pdev);
822 		cptvf_sw_cleanup(cptvf);
823 		pci_set_drvdata(pdev, NULL);
824 		pci_release_regions(pdev);
825 		pci_disable_device(pdev);
826 		cvm_crypto_exit();
827 	}
828 }
829 
cptvf_shutdown(struct pci_dev * pdev)830 static void cptvf_shutdown(struct pci_dev *pdev)
831 {
832 	cptvf_remove(pdev);
833 }
834 
835 /* Supported devices */
836 static const struct pci_device_id cptvf_id_table[] = {
837 	{PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
838 	{ 0, }  /* end of table */
839 };
840 
841 static struct pci_driver cptvf_pci_driver = {
842 	.name = DRV_NAME,
843 	.id_table = cptvf_id_table,
844 	.probe = cptvf_probe,
845 	.remove = cptvf_remove,
846 	.shutdown = cptvf_shutdown,
847 };
848 
849 module_pci_driver(cptvf_pci_driver);
850 
851 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
852 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
853 MODULE_LICENSE("GPL v2");
854 MODULE_VERSION(DRV_VERSION);
855 MODULE_DEVICE_TABLE(pci, cptvf_id_table);
856