1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cadence CDNSP DRD Driver.
4  *
5  * Copyright (C) 2020 Cadence.
6  *
7  * Author: Pawel Laszczak <pawell@cadence.com>
8  *
9  */
10 
11 #include <linux/moduleparam.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/module.h>
14 #include <linux/iopoll.h>
15 #include <linux/delay.h>
16 #include <linux/log2.h>
17 #include <linux/slab.h>
18 #include <linux/pci.h>
19 #include <linux/irq.h>
20 #include <linux/dmi.h>
21 
22 #include "core.h"
23 #include "gadget-export.h"
24 #include "drd.h"
25 #include "cdnsp-gadget.h"
26 #include "cdnsp-trace.h"
27 
cdnsp_port_speed(unsigned int port_status)28 unsigned int cdnsp_port_speed(unsigned int port_status)
29 {
30 	/*Detect gadget speed based on PORTSC register*/
31 	if (DEV_SUPERSPEEDPLUS(port_status))
32 		return USB_SPEED_SUPER_PLUS;
33 	else if (DEV_SUPERSPEED(port_status))
34 		return USB_SPEED_SUPER;
35 	else if (DEV_HIGHSPEED(port_status))
36 		return USB_SPEED_HIGH;
37 	else if (DEV_FULLSPEED(port_status))
38 		return USB_SPEED_FULL;
39 
40 	/* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
41 	return USB_SPEED_UNKNOWN;
42 }
43 
44 /*
45  * Given a port state, this function returns a value that would result in the
46  * port being in the same state, if the value was written to the port status
47  * control register.
48  * Save Read Only (RO) bits and save read/write bits where
49  * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
50  * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
51  */
cdnsp_port_state_to_neutral(u32 state)52 u32 cdnsp_port_state_to_neutral(u32 state)
53 {
54 	/* Save read-only status and port state. */
55 	return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
56 }
57 
58 /**
59  * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
60  *                           with capability ID id.
61  * @base: PCI MMIO registers base address.
62  * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
63  *         beginning of list)
64  * @id: Extended capability ID to search for.
65  *
66  * Returns the offset of the next matching extended capability structure.
67  * Some capabilities can occur several times,
68  * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
69  */
cdnsp_find_next_ext_cap(void __iomem * base,u32 start,int id)70 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
71 {
72 	u32 offset = start;
73 	u32 next;
74 	u32 val;
75 
76 	if (!start || start == HCC_PARAMS_OFFSET) {
77 		val = readl(base + HCC_PARAMS_OFFSET);
78 		if (val == ~0)
79 			return 0;
80 
81 		offset = HCC_EXT_CAPS(val) << 2;
82 		if (!offset)
83 			return 0;
84 	}
85 
86 	do {
87 		val = readl(base + offset);
88 		if (val == ~0)
89 			return 0;
90 
91 		if (EXT_CAPS_ID(val) == id && offset != start)
92 			return offset;
93 
94 		next = EXT_CAPS_NEXT(val);
95 		offset += next << 2;
96 	} while (next);
97 
98 	return 0;
99 }
100 
cdnsp_set_link_state(struct cdnsp_device * pdev,__le32 __iomem * port_regs,u32 link_state)101 void cdnsp_set_link_state(struct cdnsp_device *pdev,
102 			  __le32 __iomem *port_regs,
103 			  u32 link_state)
104 {
105 	int port_num = 0xFF;
106 	u32 temp;
107 
108 	temp = readl(port_regs);
109 	temp = cdnsp_port_state_to_neutral(temp);
110 	temp |= PORT_WKCONN_E | PORT_WKDISC_E;
111 	writel(temp, port_regs);
112 
113 	temp &= ~PORT_PLS_MASK;
114 	temp |= PORT_LINK_STROBE | link_state;
115 
116 	if (pdev->active_port)
117 		port_num = pdev->active_port->port_num;
118 
119 	trace_cdnsp_handle_port_status(port_num, readl(port_regs));
120 	writel(temp, port_regs);
121 	trace_cdnsp_link_state_changed(port_num, readl(port_regs));
122 }
123 
cdnsp_disable_port(struct cdnsp_device * pdev,__le32 __iomem * port_regs)124 static void cdnsp_disable_port(struct cdnsp_device *pdev,
125 			       __le32 __iomem *port_regs)
126 {
127 	u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
128 
129 	writel(temp | PORT_PED, port_regs);
130 }
131 
cdnsp_clear_port_change_bit(struct cdnsp_device * pdev,__le32 __iomem * port_regs)132 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
133 					__le32 __iomem *port_regs)
134 {
135 	u32 portsc = readl(port_regs);
136 
137 	writel(cdnsp_port_state_to_neutral(portsc) |
138 	       (portsc & PORT_CHANGE_BITS), port_regs);
139 }
140 
cdnsp_set_chicken_bits_2(struct cdnsp_device * pdev,u32 bit)141 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
142 {
143 	__le32 __iomem *reg;
144 	void __iomem *base;
145 	u32 offset = 0;
146 
147 	base = &pdev->cap_regs->hc_capbase;
148 	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
149 	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
150 
151 	bit = readl(reg) | bit;
152 	writel(bit, reg);
153 }
154 
cdnsp_clear_chicken_bits_2(struct cdnsp_device * pdev,u32 bit)155 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
156 {
157 	__le32 __iomem *reg;
158 	void __iomem *base;
159 	u32 offset = 0;
160 
161 	base = &pdev->cap_regs->hc_capbase;
162 	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
163 	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
164 
165 	bit = readl(reg) & ~bit;
166 	writel(bit, reg);
167 }
168 
169 /*
170  * Disable interrupts and begin the controller halting process.
171  */
cdnsp_quiesce(struct cdnsp_device * pdev)172 static void cdnsp_quiesce(struct cdnsp_device *pdev)
173 {
174 	u32 halted;
175 	u32 mask;
176 	u32 cmd;
177 
178 	mask = ~(u32)(CDNSP_IRQS);
179 
180 	halted = readl(&pdev->op_regs->status) & STS_HALT;
181 	if (!halted)
182 		mask &= ~(CMD_R_S | CMD_DEVEN);
183 
184 	cmd = readl(&pdev->op_regs->command);
185 	cmd &= mask;
186 	writel(cmd, &pdev->op_regs->command);
187 }
188 
189 /*
190  * Force controller into halt state.
191  *
192  * Disable any IRQs and clear the run/stop bit.
193  * Controller will complete any current and actively pipelined transactions, and
194  * should halt within 16 ms of the run/stop bit being cleared.
195  * Read controller Halted bit in the status register to see when the
196  * controller is finished.
197  */
cdnsp_halt(struct cdnsp_device * pdev)198 int cdnsp_halt(struct cdnsp_device *pdev)
199 {
200 	int ret;
201 	u32 val;
202 
203 	cdnsp_quiesce(pdev);
204 
205 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
206 					val & STS_HALT, 1,
207 					CDNSP_MAX_HALT_USEC);
208 	if (ret) {
209 		dev_err(pdev->dev, "ERROR: Device halt failed\n");
210 		return ret;
211 	}
212 
213 	pdev->cdnsp_state |= CDNSP_STATE_HALTED;
214 
215 	return 0;
216 }
217 
218 /*
219  * device controller died, register read returns 0xffffffff, or command never
220  * ends.
221  */
cdnsp_died(struct cdnsp_device * pdev)222 void cdnsp_died(struct cdnsp_device *pdev)
223 {
224 	dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
225 	pdev->cdnsp_state |= CDNSP_STATE_DYING;
226 	cdnsp_halt(pdev);
227 }
228 
229 /*
230  * Set the run bit and wait for the device to be running.
231  */
cdnsp_start(struct cdnsp_device * pdev)232 static int cdnsp_start(struct cdnsp_device *pdev)
233 {
234 	u32 temp;
235 	int ret;
236 
237 	temp = readl(&pdev->op_regs->command);
238 	temp |= (CMD_R_S | CMD_DEVEN);
239 	writel(temp, &pdev->op_regs->command);
240 
241 	pdev->cdnsp_state = 0;
242 
243 	/*
244 	 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
245 	 * running.
246 	 */
247 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
248 					!(temp & STS_HALT), 1,
249 					CDNSP_MAX_HALT_USEC);
250 	if (ret) {
251 		pdev->cdnsp_state = CDNSP_STATE_DYING;
252 		dev_err(pdev->dev, "ERROR: Controller run failed\n");
253 	}
254 
255 	return ret;
256 }
257 
258 /*
259  * Reset a halted controller.
260  *
261  * This resets pipelines, timers, counters, state machines, etc.
262  * Transactions will be terminated immediately, and operational registers
263  * will be set to their defaults.
264  */
cdnsp_reset(struct cdnsp_device * pdev)265 int cdnsp_reset(struct cdnsp_device *pdev)
266 {
267 	u32 command;
268 	u32 temp;
269 	int ret;
270 
271 	temp = readl(&pdev->op_regs->status);
272 
273 	if (temp == ~(u32)0) {
274 		dev_err(pdev->dev, "Device not accessible, reset failed.\n");
275 		return -ENODEV;
276 	}
277 
278 	if ((temp & STS_HALT) == 0) {
279 		dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
280 		return -EINVAL;
281 	}
282 
283 	command = readl(&pdev->op_regs->command);
284 	command |= CMD_RESET;
285 	writel(command, &pdev->op_regs->command);
286 
287 	ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
288 					!(temp & CMD_RESET), 1,
289 					10 * 1000);
290 	if (ret) {
291 		dev_err(pdev->dev, "ERROR: Controller reset failed\n");
292 		return ret;
293 	}
294 
295 	/*
296 	 * CDNSP cannot write any doorbells or operational registers other
297 	 * than status until the "Controller Not Ready" flag is cleared.
298 	 */
299 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
300 					!(temp & STS_CNR), 1,
301 					10 * 1000);
302 
303 	if (ret) {
304 		dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
305 		return ret;
306 	}
307 
308 	dev_dbg(pdev->dev, "Controller ready to work");
309 
310 	return ret;
311 }
312 
313 /*
314  * cdnsp_get_endpoint_index - Find the index for an endpoint given its
315  * descriptor.Use the return value to right shift 1 for the bitmask.
316  *
317  * Index = (epnum * 2) + direction - 1,
318  * where direction = 0 for OUT, 1 for IN.
319  * For control endpoints, the IN index is used (OUT index is unused), so
320  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
321  */
322 static unsigned int
cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor * desc)323 	cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
324 {
325 	unsigned int index = (unsigned int)usb_endpoint_num(desc);
326 
327 	if (usb_endpoint_xfer_control(desc))
328 		return index * 2;
329 
330 	return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
331 }
332 
333 /*
334  * Find the flag for this endpoint (for use in the control context). Use the
335  * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
336  * bit 1, etc.
337  */
338 static unsigned int
cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor * desc)339 	cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
340 {
341 	return 1 << (cdnsp_get_endpoint_index(desc) + 1);
342 }
343 
cdnsp_ep_enqueue(struct cdnsp_ep * pep,struct cdnsp_request * preq)344 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
345 {
346 	struct cdnsp_device *pdev = pep->pdev;
347 	struct usb_request *request;
348 	int ret;
349 
350 	if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
351 		trace_cdnsp_request_enqueue_busy(preq);
352 		return -EBUSY;
353 	}
354 
355 	request = &preq->request;
356 	request->actual = 0;
357 	request->status = -EINPROGRESS;
358 	preq->direction = pep->direction;
359 	preq->epnum = pep->number;
360 	preq->td.drbl = 0;
361 
362 	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
363 	if (ret) {
364 		trace_cdnsp_request_enqueue_error(preq);
365 		return ret;
366 	}
367 
368 	list_add_tail(&preq->list, &pep->pending_list);
369 
370 	trace_cdnsp_request_enqueue(preq);
371 
372 	switch (usb_endpoint_type(pep->endpoint.desc)) {
373 	case USB_ENDPOINT_XFER_CONTROL:
374 		ret = cdnsp_queue_ctrl_tx(pdev, preq);
375 		break;
376 	case USB_ENDPOINT_XFER_BULK:
377 	case USB_ENDPOINT_XFER_INT:
378 		ret = cdnsp_queue_bulk_tx(pdev, preq);
379 		break;
380 	case USB_ENDPOINT_XFER_ISOC:
381 		ret = cdnsp_queue_isoc_tx_prepare(pdev, preq);
382 	}
383 
384 	if (ret)
385 		goto unmap;
386 
387 	return 0;
388 
389 unmap:
390 	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
391 					pep->direction);
392 	list_del(&preq->list);
393 	trace_cdnsp_request_enqueue_error(preq);
394 
395 	return ret;
396 }
397 
398 /*
399  * Remove the request's TD from the endpoint ring. This may cause the
400  * controller to stop USB transfers, potentially stopping in the middle of a
401  * TRB buffer. The controller should pick up where it left off in the TD,
402  * unless a Set Transfer Ring Dequeue Pointer is issued.
403  *
404  * The TRBs that make up the buffers for the canceled request will be "removed"
405  * from the ring. Since the ring is a contiguous structure, they can't be
406  * physically removed. Instead, there are two options:
407  *
408  *  1) If the controller is in the middle of processing the request to be
409  *     canceled, we simply move the ring's dequeue pointer past those TRBs
410  *     using the Set Transfer Ring Dequeue Pointer command. This will be
411  *     the common case, when drivers timeout on the last submitted request
412  *     and attempt to cancel.
413  *
414  *  2) If the controller is in the middle of a different TD, we turn the TRBs
415  *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
416  *     The controller will need to invalidate the any TRBs it has cached after
417  *     the stop endpoint command.
418  *
419  *  3) The TD may have completed by the time the Stop Endpoint Command
420  *     completes, so software needs to handle that case too.
421  *
422  */
cdnsp_ep_dequeue(struct cdnsp_ep * pep,struct cdnsp_request * preq)423 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
424 {
425 	struct cdnsp_device *pdev = pep->pdev;
426 	int ret_stop = 0;
427 	int ret_rem;
428 
429 	trace_cdnsp_request_dequeue(preq);
430 
431 	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
432 		ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
433 
434 	ret_rem = cdnsp_remove_request(pdev, preq, pep);
435 
436 	return ret_rem ? ret_rem : ret_stop;
437 }
438 
cdnsp_zero_in_ctx(struct cdnsp_device * pdev)439 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
440 {
441 	struct cdnsp_input_control_ctx *ctrl_ctx;
442 	struct cdnsp_slot_ctx *slot_ctx;
443 	struct cdnsp_ep_ctx *ep_ctx;
444 	int i;
445 
446 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
447 
448 	/*
449 	 * When a device's add flag and drop flag are zero, any subsequent
450 	 * configure endpoint command will leave that endpoint's state
451 	 * untouched. Make sure we don't leave any old state in the input
452 	 * endpoint contexts.
453 	 */
454 	ctrl_ctx->drop_flags = 0;
455 	ctrl_ctx->add_flags = 0;
456 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
457 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
458 
459 	/* Endpoint 0 is always valid */
460 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
461 	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
462 		ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
463 		ep_ctx->ep_info = 0;
464 		ep_ctx->ep_info2 = 0;
465 		ep_ctx->deq = 0;
466 		ep_ctx->tx_info = 0;
467 	}
468 }
469 
470 /* Issue a configure endpoint command and wait for it to finish. */
cdnsp_configure_endpoint(struct cdnsp_device * pdev)471 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
472 {
473 	int ret;
474 
475 	cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
476 	cdnsp_ring_cmd_db(pdev);
477 	ret = cdnsp_wait_for_cmd_compl(pdev);
478 	if (ret) {
479 		dev_err(pdev->dev,
480 			"ERR: unexpected command completion code 0x%x.\n", ret);
481 		return -EINVAL;
482 	}
483 
484 	return ret;
485 }
486 
cdnsp_invalidate_ep_events(struct cdnsp_device * pdev,struct cdnsp_ep * pep)487 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
488 				       struct cdnsp_ep *pep)
489 {
490 	struct cdnsp_segment *segment;
491 	union cdnsp_trb *event;
492 	u32 cycle_state;
493 	u32  data;
494 
495 	event = pdev->event_ring->dequeue;
496 	segment = pdev->event_ring->deq_seg;
497 	cycle_state = pdev->event_ring->cycle_state;
498 
499 	while (1) {
500 		data = le32_to_cpu(event->trans_event.flags);
501 
502 		/* Check the owner of the TRB. */
503 		if ((data & TRB_CYCLE) != cycle_state)
504 			break;
505 
506 		if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
507 		    TRB_TO_EP_ID(data) == (pep->idx + 1)) {
508 			data |= TRB_EVENT_INVALIDATE;
509 			event->trans_event.flags = cpu_to_le32(data);
510 		}
511 
512 		if (cdnsp_last_trb_on_seg(segment, event)) {
513 			cycle_state ^= 1;
514 			segment = pdev->event_ring->deq_seg->next;
515 			event = segment->trbs;
516 		} else {
517 			event++;
518 		}
519 	}
520 }
521 
cdnsp_wait_for_cmd_compl(struct cdnsp_device * pdev)522 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
523 {
524 	struct cdnsp_segment *event_deq_seg;
525 	union cdnsp_trb *cmd_trb;
526 	dma_addr_t cmd_deq_dma;
527 	union cdnsp_trb *event;
528 	u32 cycle_state;
529 	int ret, val;
530 	u64 cmd_dma;
531 	u32  flags;
532 
533 	cmd_trb = pdev->cmd.command_trb;
534 	pdev->cmd.status = 0;
535 
536 	trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
537 
538 	ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
539 					!CMD_RING_BUSY(val), 1,
540 					CDNSP_CMD_TIMEOUT);
541 	if (ret) {
542 		dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
543 		trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
544 		pdev->cdnsp_state = CDNSP_STATE_DYING;
545 		return -ETIMEDOUT;
546 	}
547 
548 	event = pdev->event_ring->dequeue;
549 	event_deq_seg = pdev->event_ring->deq_seg;
550 	cycle_state = pdev->event_ring->cycle_state;
551 
552 	cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
553 	if (!cmd_deq_dma)
554 		return -EINVAL;
555 
556 	while (1) {
557 		flags = le32_to_cpu(event->event_cmd.flags);
558 
559 		/* Check the owner of the TRB. */
560 		if ((flags & TRB_CYCLE) != cycle_state)
561 			return -EINVAL;
562 
563 		cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
564 
565 		/*
566 		 * Check whether the completion event is for last queued
567 		 * command.
568 		 */
569 		if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
570 		    cmd_dma != (u64)cmd_deq_dma) {
571 			if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
572 				event++;
573 				continue;
574 			}
575 
576 			if (cdnsp_last_trb_on_ring(pdev->event_ring,
577 						   event_deq_seg, event))
578 				cycle_state ^= 1;
579 
580 			event_deq_seg = event_deq_seg->next;
581 			event = event_deq_seg->trbs;
582 			continue;
583 		}
584 
585 		trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
586 
587 		pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
588 		if (pdev->cmd.status == COMP_SUCCESS)
589 			return 0;
590 
591 		return -pdev->cmd.status;
592 	}
593 }
594 
cdnsp_halt_endpoint(struct cdnsp_device * pdev,struct cdnsp_ep * pep,int value)595 int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
596 			struct cdnsp_ep *pep,
597 			int value)
598 {
599 	int ret;
600 
601 	trace_cdnsp_ep_halt(value ? "Set" : "Clear");
602 
603 	ret = cdnsp_cmd_stop_ep(pdev, pep);
604 	if (ret)
605 		return ret;
606 
607 	if (value) {
608 		if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
609 			cdnsp_queue_halt_endpoint(pdev, pep->idx);
610 			cdnsp_ring_cmd_db(pdev);
611 			ret = cdnsp_wait_for_cmd_compl(pdev);
612 		}
613 
614 		pep->ep_state |= EP_HALTED;
615 	} else {
616 		cdnsp_queue_reset_ep(pdev, pep->idx);
617 		cdnsp_ring_cmd_db(pdev);
618 		ret = cdnsp_wait_for_cmd_compl(pdev);
619 		trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
620 
621 		if (ret)
622 			return ret;
623 
624 		pep->ep_state &= ~EP_HALTED;
625 
626 		if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
627 			cdnsp_ring_doorbell_for_active_rings(pdev, pep);
628 
629 		pep->ep_state &= ~EP_WEDGE;
630 	}
631 
632 	return 0;
633 }
634 
cdnsp_update_eps_configuration(struct cdnsp_device * pdev,struct cdnsp_ep * pep)635 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
636 					  struct cdnsp_ep *pep)
637 {
638 	struct cdnsp_input_control_ctx *ctrl_ctx;
639 	struct cdnsp_slot_ctx *slot_ctx;
640 	int ret = 0;
641 	u32 ep_sts;
642 	int i;
643 
644 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
645 
646 	/* Don't issue the command if there's no endpoints to update. */
647 	if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
648 		return 0;
649 
650 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
651 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
652 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
653 
654 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
655 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
656 	for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
657 		__le32 le32 = cpu_to_le32(BIT(i));
658 
659 		if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
660 		    (ctrl_ctx->add_flags & le32) || i == 1) {
661 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
662 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
663 			break;
664 		}
665 	}
666 
667 	ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
668 
669 	if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
670 	     ep_sts == EP_STATE_DISABLED) ||
671 	    (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
672 		ret = cdnsp_configure_endpoint(pdev);
673 
674 	trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
675 	trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
676 
677 	cdnsp_zero_in_ctx(pdev);
678 
679 	return ret;
680 }
681 
682 /*
683  * This submits a Reset Device Command, which will set the device state to 0,
684  * set the device address to 0, and disable all the endpoints except the default
685  * control endpoint. The USB core should come back and call
686  * cdnsp_setup_device(), and then re-set up the configuration.
687  */
cdnsp_reset_device(struct cdnsp_device * pdev)688 int cdnsp_reset_device(struct cdnsp_device *pdev)
689 {
690 	struct cdnsp_slot_ctx *slot_ctx;
691 	int slot_state;
692 	int ret, i;
693 
694 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
695 	slot_ctx->dev_info = 0;
696 	pdev->device_address = 0;
697 
698 	/* If device is not setup, there is no point in resetting it. */
699 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
700 	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
701 	trace_cdnsp_reset_device(slot_ctx);
702 
703 	if (slot_state <= SLOT_STATE_DEFAULT &&
704 	    pdev->eps[0].ep_state & EP_HALTED) {
705 		cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
706 	}
707 
708 	/*
709 	 * During Reset Device command controller shall transition the
710 	 * endpoint ep0 to the Running State.
711 	 */
712 	pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
713 	pdev->eps[0].ep_state |= EP_ENABLED;
714 
715 	if (slot_state <= SLOT_STATE_DEFAULT)
716 		return 0;
717 
718 	cdnsp_queue_reset_device(pdev);
719 	cdnsp_ring_cmd_db(pdev);
720 	ret = cdnsp_wait_for_cmd_compl(pdev);
721 
722 	/*
723 	 * After Reset Device command all not default endpoints
724 	 * are in Disabled state.
725 	 */
726 	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
727 		pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
728 
729 	trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
730 
731 	if (ret)
732 		dev_err(pdev->dev, "Reset device failed with error code %d",
733 			ret);
734 
735 	return ret;
736 }
737 
738 /*
739  * Sets the MaxPStreams field and the Linear Stream Array field.
740  * Sets the dequeue pointer to the stream context array.
741  */
cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device * pdev,struct cdnsp_ep_ctx * ep_ctx,struct cdnsp_stream_info * stream_info)742 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
743 					     struct cdnsp_ep_ctx *ep_ctx,
744 					     struct cdnsp_stream_info *stream_info)
745 {
746 	u32 max_primary_streams;
747 
748 	/* MaxPStreams is the number of stream context array entries, not the
749 	 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
750 	 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
751 	 */
752 	max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
753 	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
754 	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
755 				       | EP_HAS_LSA);
756 	ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
757 }
758 
759 /*
760  * The drivers use this function to prepare a bulk endpoints to use streams.
761  *
762  * Don't allow the call to succeed if endpoint only supports one stream
763  * (which means it doesn't support streams at all).
764  */
cdnsp_alloc_streams(struct cdnsp_device * pdev,struct cdnsp_ep * pep)765 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
766 {
767 	unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
768 	unsigned int num_stream_ctxs;
769 	int ret;
770 
771 	if (num_streams ==  0)
772 		return 0;
773 
774 	if (num_streams > STREAM_NUM_STREAMS)
775 		return -EINVAL;
776 
777 	/*
778 	 * Add two to the number of streams requested to account for
779 	 * stream 0 that is reserved for controller usage and one additional
780 	 * for TASK SET FULL response.
781 	 */
782 	num_streams += 2;
783 
784 	/* The stream context array size must be a power of two */
785 	num_stream_ctxs = roundup_pow_of_two(num_streams);
786 
787 	trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
788 
789 	ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
790 	if (ret)
791 		return ret;
792 
793 	cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
794 
795 	pep->ep_state |= EP_HAS_STREAMS;
796 	pep->stream_info.td_count = 0;
797 	pep->stream_info.first_prime_det = 0;
798 
799 	/* Subtract 1 for stream 0, which drivers can't use. */
800 	return num_streams - 1;
801 }
802 
cdnsp_disable_slot(struct cdnsp_device * pdev)803 int cdnsp_disable_slot(struct cdnsp_device *pdev)
804 {
805 	int ret;
806 
807 	cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
808 	cdnsp_ring_cmd_db(pdev);
809 	ret = cdnsp_wait_for_cmd_compl(pdev);
810 
811 	pdev->slot_id = 0;
812 	pdev->active_port = NULL;
813 
814 	trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
815 
816 	memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
817 	memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
818 
819 	return ret;
820 }
821 
cdnsp_enable_slot(struct cdnsp_device * pdev)822 int cdnsp_enable_slot(struct cdnsp_device *pdev)
823 {
824 	struct cdnsp_slot_ctx *slot_ctx;
825 	int slot_state;
826 	int ret;
827 
828 	/* If device is not setup, there is no point in resetting it */
829 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
830 	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
831 
832 	if (slot_state != SLOT_STATE_DISABLED)
833 		return 0;
834 
835 	cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
836 	cdnsp_ring_cmd_db(pdev);
837 	ret = cdnsp_wait_for_cmd_compl(pdev);
838 	if (ret)
839 		goto show_trace;
840 
841 	pdev->slot_id = 1;
842 
843 show_trace:
844 	trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
845 
846 	return ret;
847 }
848 
849 /*
850  * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
851  * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
852  */
cdnsp_setup_device(struct cdnsp_device * pdev,enum cdnsp_setup_dev setup)853 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
854 {
855 	struct cdnsp_input_control_ctx *ctrl_ctx;
856 	struct cdnsp_slot_ctx *slot_ctx;
857 	int dev_state = 0;
858 	int ret;
859 
860 	if (!pdev->slot_id) {
861 		trace_cdnsp_slot_id("incorrect");
862 		return -EINVAL;
863 	}
864 
865 	if (!pdev->active_port->port_num)
866 		return -EINVAL;
867 
868 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
869 	dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
870 
871 	if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
872 		trace_cdnsp_slot_already_in_default(slot_ctx);
873 		return 0;
874 	}
875 
876 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
877 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
878 
879 	if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
880 		ret = cdnsp_setup_addressable_priv_dev(pdev);
881 		if (ret)
882 			return ret;
883 	}
884 
885 	cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
886 
887 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
888 	ctrl_ctx->drop_flags = 0;
889 
890 	trace_cdnsp_setup_device_slot(slot_ctx);
891 
892 	cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
893 	cdnsp_ring_cmd_db(pdev);
894 	ret = cdnsp_wait_for_cmd_compl(pdev);
895 
896 	trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
897 
898 	/* Zero the input context control for later use. */
899 	ctrl_ctx->add_flags = 0;
900 	ctrl_ctx->drop_flags = 0;
901 
902 	return ret;
903 }
904 
cdnsp_set_usb2_hardware_lpm(struct cdnsp_device * pdev,struct usb_request * req,int enable)905 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
906 				 struct usb_request *req,
907 				 int enable)
908 {
909 	if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
910 		return;
911 
912 	trace_cdnsp_lpm(enable);
913 
914 	if (enable)
915 		writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
916 		       &pdev->active_port->regs->portpmsc);
917 	else
918 		writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
919 }
920 
cdnsp_get_frame(struct cdnsp_device * pdev)921 static int cdnsp_get_frame(struct cdnsp_device *pdev)
922 {
923 	return readl(&pdev->run_regs->microframe_index) >> 3;
924 }
925 
cdnsp_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)926 static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
927 				  const struct usb_endpoint_descriptor *desc)
928 {
929 	struct cdnsp_input_control_ctx *ctrl_ctx;
930 	struct cdnsp_device *pdev;
931 	struct cdnsp_ep *pep;
932 	unsigned long flags;
933 	u32 added_ctxs;
934 	int ret;
935 
936 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
937 	    !desc->wMaxPacketSize)
938 		return -EINVAL;
939 
940 	pep = to_cdnsp_ep(ep);
941 	pdev = pep->pdev;
942 	pep->ep_state &= ~EP_UNCONFIGURED;
943 
944 	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
945 			  "%s is already enabled\n", pep->name))
946 		return 0;
947 
948 	spin_lock_irqsave(&pdev->lock, flags);
949 
950 	added_ctxs = cdnsp_get_endpoint_flag(desc);
951 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
952 		dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
953 		ret = -EINVAL;
954 		goto unlock;
955 	}
956 
957 	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
958 
959 	if (pdev->gadget.speed == USB_SPEED_FULL) {
960 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
961 			pep->interval = desc->bInterval << 3;
962 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
963 			pep->interval = BIT(desc->bInterval - 1) << 3;
964 	}
965 
966 	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
967 		if (pep->interval > BIT(12)) {
968 			dev_err(pdev->dev, "bInterval %d not supported\n",
969 				desc->bInterval);
970 			ret = -EINVAL;
971 			goto unlock;
972 		}
973 		cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
974 	}
975 
976 	ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
977 	if (ret)
978 		goto unlock;
979 
980 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
981 	ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
982 	ctrl_ctx->drop_flags = 0;
983 
984 	ret = cdnsp_update_eps_configuration(pdev, pep);
985 	if (ret) {
986 		cdnsp_free_endpoint_rings(pdev, pep);
987 		goto unlock;
988 	}
989 
990 	pep->ep_state |= EP_ENABLED;
991 	pep->ep_state &= ~EP_STOPPED;
992 
993 unlock:
994 	trace_cdnsp_ep_enable_end(pep, 0);
995 	spin_unlock_irqrestore(&pdev->lock, flags);
996 
997 	return ret;
998 }
999 
cdnsp_gadget_ep_disable(struct usb_ep * ep)1000 static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1001 {
1002 	struct cdnsp_input_control_ctx *ctrl_ctx;
1003 	struct cdnsp_request *preq;
1004 	struct cdnsp_device *pdev;
1005 	struct cdnsp_ep *pep;
1006 	unsigned long flags;
1007 	u32 drop_flag;
1008 	int ret = 0;
1009 
1010 	if (!ep)
1011 		return -EINVAL;
1012 
1013 	pep = to_cdnsp_ep(ep);
1014 	pdev = pep->pdev;
1015 
1016 	spin_lock_irqsave(&pdev->lock, flags);
1017 
1018 	if (!(pep->ep_state & EP_ENABLED)) {
1019 		dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1020 		ret = -EINVAL;
1021 		goto finish;
1022 	}
1023 
1024 	pep->ep_state |= EP_DIS_IN_RROGRESS;
1025 
1026 	/* Endpoint was unconfigured by Reset Device command. */
1027 	if (!(pep->ep_state & EP_UNCONFIGURED)) {
1028 		cdnsp_cmd_stop_ep(pdev, pep);
1029 		cdnsp_cmd_flush_ep(pdev, pep);
1030 	}
1031 
1032 	/* Remove all queued USB requests. */
1033 	while (!list_empty(&pep->pending_list)) {
1034 		preq = next_request(&pep->pending_list);
1035 		cdnsp_ep_dequeue(pep, preq);
1036 	}
1037 
1038 	cdnsp_invalidate_ep_events(pdev, pep);
1039 
1040 	pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1041 	drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1042 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1043 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1044 	ctrl_ctx->add_flags = 0;
1045 
1046 	cdnsp_endpoint_zero(pdev, pep);
1047 
1048 	if (!(pep->ep_state & EP_UNCONFIGURED))
1049 		ret = cdnsp_update_eps_configuration(pdev, pep);
1050 
1051 	cdnsp_free_endpoint_rings(pdev, pep);
1052 
1053 	pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1054 	pep->ep_state |= EP_STOPPED;
1055 
1056 finish:
1057 	trace_cdnsp_ep_disable_end(pep, 0);
1058 	spin_unlock_irqrestore(&pdev->lock, flags);
1059 
1060 	return ret;
1061 }
1062 
cdnsp_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1063 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1064 							 gfp_t gfp_flags)
1065 {
1066 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1067 	struct cdnsp_request *preq;
1068 
1069 	preq = kzalloc(sizeof(*preq), gfp_flags);
1070 	if (!preq)
1071 		return NULL;
1072 
1073 	preq->epnum = pep->number;
1074 	preq->pep = pep;
1075 
1076 	trace_cdnsp_alloc_request(preq);
1077 
1078 	return &preq->request;
1079 }
1080 
cdnsp_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1081 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1082 					 struct usb_request *request)
1083 {
1084 	struct cdnsp_request *preq = to_cdnsp_request(request);
1085 
1086 	trace_cdnsp_free_request(preq);
1087 	kfree(preq);
1088 }
1089 
cdnsp_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1090 static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1091 				 struct usb_request *request,
1092 				 gfp_t gfp_flags)
1093 {
1094 	struct cdnsp_request *preq;
1095 	struct cdnsp_device *pdev;
1096 	struct cdnsp_ep *pep;
1097 	unsigned long flags;
1098 	int ret;
1099 
1100 	if (!request || !ep)
1101 		return -EINVAL;
1102 
1103 	pep = to_cdnsp_ep(ep);
1104 	pdev = pep->pdev;
1105 
1106 	if (!(pep->ep_state & EP_ENABLED)) {
1107 		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1108 			pep->name);
1109 		return -EINVAL;
1110 	}
1111 
1112 	preq = to_cdnsp_request(request);
1113 	spin_lock_irqsave(&pdev->lock, flags);
1114 	ret = cdnsp_ep_enqueue(pep, preq);
1115 	spin_unlock_irqrestore(&pdev->lock, flags);
1116 
1117 	return ret;
1118 }
1119 
cdnsp_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)1120 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1121 				   struct usb_request *request)
1122 {
1123 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1124 	struct cdnsp_device *pdev = pep->pdev;
1125 	unsigned long flags;
1126 	int ret;
1127 
1128 	if (!pep->endpoint.desc) {
1129 		dev_err(pdev->dev,
1130 			"%s: can't dequeue to disabled endpoint\n",
1131 			pep->name);
1132 		return -ESHUTDOWN;
1133 	}
1134 
1135 	/* Requests has been dequeued during disabling endpoint. */
1136 	if (!(pep->ep_state & EP_ENABLED))
1137 		return 0;
1138 
1139 	spin_lock_irqsave(&pdev->lock, flags);
1140 	ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1141 	spin_unlock_irqrestore(&pdev->lock, flags);
1142 
1143 	return ret;
1144 }
1145 
cdnsp_gadget_ep_set_halt(struct usb_ep * ep,int value)1146 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1147 {
1148 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1149 	struct cdnsp_device *pdev = pep->pdev;
1150 	struct cdnsp_request *preq;
1151 	unsigned long flags;
1152 	int ret;
1153 
1154 	spin_lock_irqsave(&pdev->lock, flags);
1155 
1156 	preq = next_request(&pep->pending_list);
1157 	if (value) {
1158 		if (preq) {
1159 			trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1160 			ret = -EAGAIN;
1161 			goto done;
1162 		}
1163 	}
1164 
1165 	ret = cdnsp_halt_endpoint(pdev, pep, value);
1166 
1167 done:
1168 	spin_unlock_irqrestore(&pdev->lock, flags);
1169 	return ret;
1170 }
1171 
cdnsp_gadget_ep_set_wedge(struct usb_ep * ep)1172 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1173 {
1174 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1175 	struct cdnsp_device *pdev = pep->pdev;
1176 	unsigned long flags;
1177 	int ret;
1178 
1179 	spin_lock_irqsave(&pdev->lock, flags);
1180 	pep->ep_state |= EP_WEDGE;
1181 	ret = cdnsp_halt_endpoint(pdev, pep, 1);
1182 	spin_unlock_irqrestore(&pdev->lock, flags);
1183 
1184 	return ret;
1185 }
1186 
1187 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1188 	.enable		= cdnsp_gadget_ep_enable,
1189 	.disable	= cdnsp_gadget_ep_disable,
1190 	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1191 	.free_request	= cdnsp_gadget_ep_free_request,
1192 	.queue		= cdnsp_gadget_ep_queue,
1193 	.dequeue	= cdnsp_gadget_ep_dequeue,
1194 	.set_halt	= cdnsp_gadget_ep_set_halt,
1195 	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1196 };
1197 
1198 static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1199 	.enable		= cdnsp_gadget_ep_enable,
1200 	.disable	= cdnsp_gadget_ep_disable,
1201 	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1202 	.free_request	= cdnsp_gadget_ep_free_request,
1203 	.queue		= cdnsp_gadget_ep_queue,
1204 	.dequeue	= cdnsp_gadget_ep_dequeue,
1205 	.set_halt	= cdnsp_gadget_ep_set_halt,
1206 	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1207 };
1208 
cdnsp_gadget_giveback(struct cdnsp_ep * pep,struct cdnsp_request * preq,int status)1209 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1210 			   struct cdnsp_request *preq,
1211 			   int status)
1212 {
1213 	struct cdnsp_device *pdev = pep->pdev;
1214 
1215 	list_del(&preq->list);
1216 
1217 	if (preq->request.status == -EINPROGRESS)
1218 		preq->request.status = status;
1219 
1220 	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1221 					preq->direction);
1222 
1223 	trace_cdnsp_request_giveback(preq);
1224 
1225 	if (preq != &pdev->ep0_preq) {
1226 		spin_unlock(&pdev->lock);
1227 		usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1228 		spin_lock(&pdev->lock);
1229 	}
1230 }
1231 
1232 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1233 	.bLength =		USB_DT_ENDPOINT_SIZE,
1234 	.bDescriptorType =	USB_DT_ENDPOINT,
1235 	.bmAttributes =		USB_ENDPOINT_XFER_CONTROL,
1236 };
1237 
cdnsp_run(struct cdnsp_device * pdev,enum usb_device_speed speed)1238 static int cdnsp_run(struct cdnsp_device *pdev,
1239 		     enum usb_device_speed speed)
1240 {
1241 	u32 fs_speed = 0;
1242 	u32 temp;
1243 	int ret;
1244 
1245 	temp = readl(&pdev->ir_set->irq_control);
1246 	temp &= ~IMOD_INTERVAL_MASK;
1247 	temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1248 	writel(temp, &pdev->ir_set->irq_control);
1249 
1250 	temp = readl(&pdev->port3x_regs->mode_addr);
1251 
1252 	switch (speed) {
1253 	case USB_SPEED_SUPER_PLUS:
1254 		temp |= CFG_3XPORT_SSP_SUPPORT;
1255 		break;
1256 	case USB_SPEED_SUPER:
1257 		temp &= ~CFG_3XPORT_SSP_SUPPORT;
1258 		break;
1259 	case USB_SPEED_HIGH:
1260 		break;
1261 	case USB_SPEED_FULL:
1262 		fs_speed = PORT_REG6_FORCE_FS;
1263 		break;
1264 	default:
1265 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1266 			speed);
1267 		fallthrough;
1268 	case USB_SPEED_UNKNOWN:
1269 		/* Default to superspeed. */
1270 		speed = USB_SPEED_SUPER;
1271 		break;
1272 	}
1273 
1274 	if (speed >= USB_SPEED_SUPER) {
1275 		writel(temp, &pdev->port3x_regs->mode_addr);
1276 		cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1277 				     XDEV_RXDETECT);
1278 	} else {
1279 		cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1280 	}
1281 
1282 	cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1283 			     XDEV_RXDETECT);
1284 
1285 	cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1286 
1287 	writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1288 
1289 	ret = cdnsp_start(pdev);
1290 	if (ret) {
1291 		ret = -ENODEV;
1292 		goto err;
1293 	}
1294 
1295 	temp = readl(&pdev->op_regs->command);
1296 	temp |= (CMD_INTE);
1297 	writel(temp, &pdev->op_regs->command);
1298 
1299 	temp = readl(&pdev->ir_set->irq_pending);
1300 	writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1301 
1302 	trace_cdnsp_init("Controller ready to work");
1303 	return 0;
1304 err:
1305 	cdnsp_halt(pdev);
1306 	return ret;
1307 }
1308 
cdnsp_gadget_udc_start(struct usb_gadget * g,struct usb_gadget_driver * driver)1309 static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1310 				  struct usb_gadget_driver *driver)
1311 {
1312 	enum usb_device_speed max_speed = driver->max_speed;
1313 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1314 	unsigned long flags;
1315 	int ret;
1316 
1317 	spin_lock_irqsave(&pdev->lock, flags);
1318 	pdev->gadget_driver = driver;
1319 
1320 	/* limit speed if necessary */
1321 	max_speed = min(driver->max_speed, g->max_speed);
1322 	ret = cdnsp_run(pdev, max_speed);
1323 
1324 	spin_unlock_irqrestore(&pdev->lock, flags);
1325 
1326 	return ret;
1327 }
1328 
1329 /*
1330  * Update Event Ring Dequeue Pointer:
1331  * - When all events have finished
1332  * - To avoid "Event Ring Full Error" condition
1333  */
cdnsp_update_erst_dequeue(struct cdnsp_device * pdev,union cdnsp_trb * event_ring_deq,u8 clear_ehb)1334 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1335 			       union cdnsp_trb *event_ring_deq,
1336 			       u8 clear_ehb)
1337 {
1338 	u64 temp_64;
1339 	dma_addr_t deq;
1340 
1341 	temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1342 
1343 	/* If necessary, update the HW's version of the event ring deq ptr. */
1344 	if (event_ring_deq != pdev->event_ring->dequeue) {
1345 		deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1346 					    pdev->event_ring->dequeue);
1347 		temp_64 &= ERST_PTR_MASK;
1348 		temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1349 	}
1350 
1351 	/* Clear the event handler busy flag (RW1C). */
1352 	if (clear_ehb)
1353 		temp_64 |= ERST_EHB;
1354 	else
1355 		temp_64 &= ~ERST_EHB;
1356 
1357 	cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1358 }
1359 
cdnsp_clear_cmd_ring(struct cdnsp_device * pdev)1360 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1361 {
1362 	struct cdnsp_segment *seg;
1363 	u64 val_64;
1364 	int i;
1365 
1366 	cdnsp_initialize_ring_info(pdev->cmd_ring);
1367 
1368 	seg = pdev->cmd_ring->first_seg;
1369 	for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1370 		memset(seg->trbs, 0,
1371 		       sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1372 		seg = seg->next;
1373 	}
1374 
1375 	/* Set the address in the Command Ring Control register. */
1376 	val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1377 	val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1378 		 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1379 		 pdev->cmd_ring->cycle_state;
1380 	cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1381 }
1382 
cdnsp_consume_all_events(struct cdnsp_device * pdev)1383 static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1384 {
1385 	struct cdnsp_segment *event_deq_seg;
1386 	union cdnsp_trb *event_ring_deq;
1387 	union cdnsp_trb *event;
1388 	u32 cycle_bit;
1389 
1390 	event_ring_deq = pdev->event_ring->dequeue;
1391 	event_deq_seg = pdev->event_ring->deq_seg;
1392 	event = pdev->event_ring->dequeue;
1393 
1394 	/* Update ring dequeue pointer. */
1395 	while (1) {
1396 		cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1397 
1398 		/* Does the controller or driver own the TRB? */
1399 		if (cycle_bit != pdev->event_ring->cycle_state)
1400 			break;
1401 
1402 		cdnsp_inc_deq(pdev, pdev->event_ring);
1403 
1404 		if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1405 			event++;
1406 			continue;
1407 		}
1408 
1409 		if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1410 					   event))
1411 			cycle_bit ^= 1;
1412 
1413 		event_deq_seg = event_deq_seg->next;
1414 		event = event_deq_seg->trbs;
1415 	}
1416 
1417 	cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1418 }
1419 
cdnsp_stop(struct cdnsp_device * pdev)1420 static void cdnsp_stop(struct cdnsp_device *pdev)
1421 {
1422 	u32 temp;
1423 
1424 	cdnsp_cmd_flush_ep(pdev, &pdev->eps[0]);
1425 
1426 	/* Remove internally queued request for ep0. */
1427 	if (!list_empty(&pdev->eps[0].pending_list)) {
1428 		struct cdnsp_request *req;
1429 
1430 		req = next_request(&pdev->eps[0].pending_list);
1431 		if (req == &pdev->ep0_preq)
1432 			cdnsp_ep_dequeue(&pdev->eps[0], req);
1433 	}
1434 
1435 	cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1436 	cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1437 	cdnsp_disable_slot(pdev);
1438 	cdnsp_halt(pdev);
1439 
1440 	temp = readl(&pdev->op_regs->status);
1441 	writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1442 	temp = readl(&pdev->ir_set->irq_pending);
1443 	writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1444 
1445 	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1446 	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1447 
1448 	/* Clear interrupt line */
1449 	temp = readl(&pdev->ir_set->irq_pending);
1450 	temp |= IMAN_IP;
1451 	writel(temp, &pdev->ir_set->irq_pending);
1452 
1453 	cdnsp_consume_all_events(pdev);
1454 	cdnsp_clear_cmd_ring(pdev);
1455 
1456 	trace_cdnsp_exit("Controller stopped.");
1457 }
1458 
1459 /*
1460  * Stop controller.
1461  * This function is called by the gadget core when the driver is removed.
1462  * Disable slot, disable IRQs, and quiesce the controller.
1463  */
cdnsp_gadget_udc_stop(struct usb_gadget * g)1464 static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1465 {
1466 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1467 	unsigned long flags;
1468 
1469 	spin_lock_irqsave(&pdev->lock, flags);
1470 	cdnsp_stop(pdev);
1471 	pdev->gadget_driver = NULL;
1472 	spin_unlock_irqrestore(&pdev->lock, flags);
1473 
1474 	return 0;
1475 }
1476 
cdnsp_gadget_get_frame(struct usb_gadget * g)1477 static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1478 {
1479 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1480 
1481 	return cdnsp_get_frame(pdev);
1482 }
1483 
__cdnsp_gadget_wakeup(struct cdnsp_device * pdev)1484 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1485 {
1486 	struct cdnsp_port_regs __iomem *port_regs;
1487 	u32 portpm, portsc;
1488 
1489 	port_regs = pdev->active_port->regs;
1490 	portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1491 
1492 	/* Remote wakeup feature is not enabled by host. */
1493 	if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1494 		portpm = readl(&port_regs->portpmsc);
1495 
1496 		if (!(portpm & PORT_RWE))
1497 			return;
1498 	}
1499 
1500 	if (portsc == XDEV_U3 && !pdev->may_wakeup)
1501 		return;
1502 
1503 	cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1504 
1505 	pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1506 }
1507 
cdnsp_gadget_wakeup(struct usb_gadget * g)1508 static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1509 {
1510 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1511 	unsigned long flags;
1512 
1513 	spin_lock_irqsave(&pdev->lock, flags);
1514 	__cdnsp_gadget_wakeup(pdev);
1515 	spin_unlock_irqrestore(&pdev->lock, flags);
1516 
1517 	return 0;
1518 }
1519 
cdnsp_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)1520 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1521 					int is_selfpowered)
1522 {
1523 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1524 	unsigned long flags;
1525 
1526 	spin_lock_irqsave(&pdev->lock, flags);
1527 	g->is_selfpowered = !!is_selfpowered;
1528 	spin_unlock_irqrestore(&pdev->lock, flags);
1529 
1530 	return 0;
1531 }
1532 
cdnsp_gadget_pullup(struct usb_gadget * gadget,int is_on)1533 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1534 {
1535 	struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1536 	struct cdns *cdns = dev_get_drvdata(pdev->dev);
1537 	unsigned long flags;
1538 
1539 	trace_cdnsp_pullup(is_on);
1540 
1541 	/*
1542 	 * Disable events handling while controller is being
1543 	 * enabled/disabled.
1544 	 */
1545 	disable_irq(cdns->dev_irq);
1546 	spin_lock_irqsave(&pdev->lock, flags);
1547 
1548 	if (!is_on) {
1549 		cdnsp_reset_device(pdev);
1550 		cdns_clear_vbus(cdns);
1551 	} else {
1552 		cdns_set_vbus(cdns);
1553 	}
1554 
1555 	spin_unlock_irqrestore(&pdev->lock, flags);
1556 	enable_irq(cdns->dev_irq);
1557 
1558 	return 0;
1559 }
1560 
1561 static const struct usb_gadget_ops cdnsp_gadget_ops = {
1562 	.get_frame		= cdnsp_gadget_get_frame,
1563 	.wakeup			= cdnsp_gadget_wakeup,
1564 	.set_selfpowered	= cdnsp_gadget_set_selfpowered,
1565 	.pullup			= cdnsp_gadget_pullup,
1566 	.udc_start		= cdnsp_gadget_udc_start,
1567 	.udc_stop		= cdnsp_gadget_udc_stop,
1568 };
1569 
cdnsp_get_ep_buffering(struct cdnsp_device * pdev,struct cdnsp_ep * pep)1570 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1571 				   struct cdnsp_ep *pep)
1572 {
1573 	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1574 	int endpoints;
1575 
1576 	reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1577 
1578 	if (!pep->direction) {
1579 		pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1580 		pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1581 		pep->buffering = (pep->buffering + 1) / 2;
1582 		pep->buffering_period = (pep->buffering_period + 1) / 2;
1583 		return;
1584 	}
1585 
1586 	endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1587 
1588 	/* Set to XBUF_TX_TAG_MASK_0 register. */
1589 	reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1590 	/* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1591 	reg += pep->number * sizeof(u32) * 2;
1592 
1593 	pep->buffering = (readl(reg) + 1) / 2;
1594 	pep->buffering_period = pep->buffering;
1595 }
1596 
cdnsp_gadget_init_endpoints(struct cdnsp_device * pdev)1597 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1598 {
1599 	int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1600 	struct cdnsp_ep *pep;
1601 	int i;
1602 
1603 	INIT_LIST_HEAD(&pdev->gadget.ep_list);
1604 
1605 	if (max_streams < STREAM_LOG_STREAMS) {
1606 		dev_err(pdev->dev, "Stream size %d not supported\n",
1607 			max_streams);
1608 		return -EINVAL;
1609 	}
1610 
1611 	max_streams = STREAM_LOG_STREAMS;
1612 
1613 	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1614 		bool direction = !(i & 1); /* Start from OUT endpoint. */
1615 		u8 epnum = ((i + 1) >> 1);
1616 
1617 		if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1618 			continue;
1619 
1620 		pep = &pdev->eps[i];
1621 		pep->pdev = pdev;
1622 		pep->number = epnum;
1623 		pep->direction = direction; /* 0 for OUT, 1 for IN. */
1624 
1625 		/*
1626 		 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1627 		 * pdev->eps[0]
1628 		 */
1629 		if (epnum == 0) {
1630 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1631 				 epnum, "BiDir");
1632 
1633 			pep->idx = 0;
1634 			usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1635 			pep->endpoint.maxburst = 1;
1636 			pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1637 			pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1638 			pep->endpoint.comp_desc = NULL;
1639 			pep->endpoint.caps.type_control = true;
1640 			pep->endpoint.caps.dir_in = true;
1641 			pep->endpoint.caps.dir_out = true;
1642 
1643 			pdev->ep0_preq.epnum = pep->number;
1644 			pdev->ep0_preq.pep = pep;
1645 			pdev->gadget.ep0 = &pep->endpoint;
1646 		} else {
1647 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1648 				 epnum, (pep->direction) ? "in" : "out");
1649 
1650 			pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1651 			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1652 
1653 			pep->endpoint.max_streams = max_streams;
1654 			pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1655 			list_add_tail(&pep->endpoint.ep_list,
1656 				      &pdev->gadget.ep_list);
1657 
1658 			pep->endpoint.caps.type_iso = true;
1659 			pep->endpoint.caps.type_bulk = true;
1660 			pep->endpoint.caps.type_int = true;
1661 
1662 			pep->endpoint.caps.dir_in = direction;
1663 			pep->endpoint.caps.dir_out = !direction;
1664 		}
1665 
1666 		pep->endpoint.name = pep->name;
1667 		pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1668 		pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1669 		cdnsp_get_ep_buffering(pdev, pep);
1670 
1671 		dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1672 			"CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1673 			"SupDir IN: %s, OUT: %s\n",
1674 			pep->name, 1024,
1675 			(pep->endpoint.caps.type_control) ? "yes" : "no",
1676 			(pep->endpoint.caps.type_int) ? "yes" : "no",
1677 			(pep->endpoint.caps.type_bulk) ? "yes" : "no",
1678 			(pep->endpoint.caps.type_iso) ? "yes" : "no",
1679 			(pep->endpoint.caps.dir_in) ? "yes" : "no",
1680 			(pep->endpoint.caps.dir_out) ? "yes" : "no");
1681 
1682 		INIT_LIST_HEAD(&pep->pending_list);
1683 	}
1684 
1685 	return 0;
1686 }
1687 
cdnsp_gadget_free_endpoints(struct cdnsp_device * pdev)1688 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1689 {
1690 	struct cdnsp_ep *pep;
1691 	int i;
1692 
1693 	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1694 		pep = &pdev->eps[i];
1695 		if (pep->number != 0 && pep->out_ctx)
1696 			list_del(&pep->endpoint.ep_list);
1697 	}
1698 }
1699 
cdnsp_disconnect_gadget(struct cdnsp_device * pdev)1700 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1701 {
1702 	pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1703 
1704 	if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1705 		spin_unlock(&pdev->lock);
1706 		pdev->gadget_driver->disconnect(&pdev->gadget);
1707 		spin_lock(&pdev->lock);
1708 	}
1709 
1710 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1711 	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1712 
1713 	pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1714 }
1715 
cdnsp_suspend_gadget(struct cdnsp_device * pdev)1716 void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1717 {
1718 	if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1719 		spin_unlock(&pdev->lock);
1720 		pdev->gadget_driver->suspend(&pdev->gadget);
1721 		spin_lock(&pdev->lock);
1722 	}
1723 }
1724 
cdnsp_resume_gadget(struct cdnsp_device * pdev)1725 void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1726 {
1727 	if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1728 		spin_unlock(&pdev->lock);
1729 		pdev->gadget_driver->resume(&pdev->gadget);
1730 		spin_lock(&pdev->lock);
1731 	}
1732 }
1733 
cdnsp_irq_reset(struct cdnsp_device * pdev)1734 void cdnsp_irq_reset(struct cdnsp_device *pdev)
1735 {
1736 	struct cdnsp_port_regs __iomem *port_regs;
1737 
1738 	cdnsp_reset_device(pdev);
1739 
1740 	port_regs = pdev->active_port->regs;
1741 	pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1742 
1743 	spin_unlock(&pdev->lock);
1744 	usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1745 	spin_lock(&pdev->lock);
1746 
1747 	switch (pdev->gadget.speed) {
1748 	case USB_SPEED_SUPER_PLUS:
1749 	case USB_SPEED_SUPER:
1750 		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1751 		pdev->gadget.ep0->maxpacket = 512;
1752 		break;
1753 	case USB_SPEED_HIGH:
1754 	case USB_SPEED_FULL:
1755 		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1756 		pdev->gadget.ep0->maxpacket = 64;
1757 		break;
1758 	default:
1759 		/* Low speed is not supported. */
1760 		dev_err(pdev->dev, "Unknown device speed\n");
1761 		break;
1762 	}
1763 
1764 	cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1765 	cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1766 	usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1767 }
1768 
cdnsp_get_rev_cap(struct cdnsp_device * pdev)1769 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1770 {
1771 	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1772 
1773 	reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1774 	pdev->rev_cap  = reg;
1775 
1776 	dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1777 		 readl(&pdev->rev_cap->ctrl_revision),
1778 		 readl(&pdev->rev_cap->rtl_revision),
1779 		 readl(&pdev->rev_cap->ep_supported),
1780 		 readl(&pdev->rev_cap->rx_buff_size),
1781 		 readl(&pdev->rev_cap->tx_buff_size));
1782 }
1783 
cdnsp_gen_setup(struct cdnsp_device * pdev)1784 static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1785 {
1786 	int ret;
1787 	u32 reg;
1788 
1789 	pdev->cap_regs = pdev->regs;
1790 	pdev->op_regs = pdev->regs +
1791 		HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1792 	pdev->run_regs = pdev->regs +
1793 		(readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1794 
1795 	/* Cache read-only capability registers */
1796 	pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1797 	pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1798 	pdev->hci_version = HC_VERSION(pdev->hcc_params);
1799 	pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1800 
1801 	cdnsp_get_rev_cap(pdev);
1802 
1803 	/* Make sure the Device Controller is halted. */
1804 	ret = cdnsp_halt(pdev);
1805 	if (ret)
1806 		return ret;
1807 
1808 	/* Reset the internal controller memory state and registers. */
1809 	ret = cdnsp_reset(pdev);
1810 	if (ret)
1811 		return ret;
1812 
1813 	/*
1814 	 * Set dma_mask and coherent_dma_mask to 64-bits,
1815 	 * if controller supports 64-bit addressing.
1816 	 */
1817 	if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1818 	    !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1819 		dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1820 		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1821 	} else {
1822 		/*
1823 		 * This is to avoid error in cases where a 32-bit USB
1824 		 * controller is used on a 64-bit capable system.
1825 		 */
1826 		ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1827 		if (ret)
1828 			return ret;
1829 
1830 		dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1831 		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1832 	}
1833 
1834 	spin_lock_init(&pdev->lock);
1835 
1836 	ret = cdnsp_mem_init(pdev);
1837 	if (ret)
1838 		return ret;
1839 
1840 	/*
1841 	 * Software workaround for U1: after transition
1842 	 * to U1 the controller starts gating clock, and in some cases,
1843 	 * it causes that controller stack.
1844 	 */
1845 	reg = readl(&pdev->port3x_regs->mode_2);
1846 	reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1847 	writel(reg, &pdev->port3x_regs->mode_2);
1848 
1849 	return 0;
1850 }
1851 
__cdnsp_gadget_init(struct cdns * cdns)1852 static int __cdnsp_gadget_init(struct cdns *cdns)
1853 {
1854 	struct cdnsp_device *pdev;
1855 	u32 max_speed;
1856 	int ret = -ENOMEM;
1857 
1858 	cdns_drd_gadget_on(cdns);
1859 
1860 	pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1861 	if (!pdev)
1862 		return -ENOMEM;
1863 
1864 	pm_runtime_get_sync(cdns->dev);
1865 
1866 	cdns->gadget_dev = pdev;
1867 	pdev->dev = cdns->dev;
1868 	pdev->regs = cdns->dev_regs;
1869 	max_speed = usb_get_maximum_speed(cdns->dev);
1870 
1871 	switch (max_speed) {
1872 	case USB_SPEED_FULL:
1873 	case USB_SPEED_HIGH:
1874 	case USB_SPEED_SUPER:
1875 	case USB_SPEED_SUPER_PLUS:
1876 		break;
1877 	default:
1878 		dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1879 		fallthrough;
1880 	case USB_SPEED_UNKNOWN:
1881 		/* Default to SSP */
1882 		max_speed = USB_SPEED_SUPER_PLUS;
1883 		break;
1884 	}
1885 
1886 	pdev->gadget.ops = &cdnsp_gadget_ops;
1887 	pdev->gadget.name = "cdnsp-gadget";
1888 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1889 	pdev->gadget.sg_supported = 1;
1890 	pdev->gadget.max_speed = max_speed;
1891 	pdev->gadget.lpm_capable = 1;
1892 
1893 	pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1894 	if (!pdev->setup_buf)
1895 		goto free_pdev;
1896 
1897 	/*
1898 	 * Controller supports not aligned buffer but it should improve
1899 	 * performance.
1900 	 */
1901 	pdev->gadget.quirk_ep_out_aligned_size = true;
1902 
1903 	ret = cdnsp_gen_setup(pdev);
1904 	if (ret) {
1905 		dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1906 		goto free_setup;
1907 	}
1908 
1909 	ret = cdnsp_gadget_init_endpoints(pdev);
1910 	if (ret) {
1911 		dev_err(pdev->dev, "failed to initialize endpoints\n");
1912 		goto halt_pdev;
1913 	}
1914 
1915 	ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1916 	if (ret) {
1917 		dev_err(pdev->dev, "failed to register udc\n");
1918 		goto free_endpoints;
1919 	}
1920 
1921 	ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1922 					cdnsp_irq_handler,
1923 					cdnsp_thread_irq_handler, IRQF_SHARED,
1924 					dev_name(pdev->dev), pdev);
1925 	if (ret)
1926 		goto del_gadget;
1927 
1928 	return 0;
1929 
1930 del_gadget:
1931 	usb_del_gadget_udc(&pdev->gadget);
1932 free_endpoints:
1933 	cdnsp_gadget_free_endpoints(pdev);
1934 halt_pdev:
1935 	cdnsp_halt(pdev);
1936 	cdnsp_reset(pdev);
1937 	cdnsp_mem_cleanup(pdev);
1938 free_setup:
1939 	kfree(pdev->setup_buf);
1940 free_pdev:
1941 	kfree(pdev);
1942 
1943 	return ret;
1944 }
1945 
cdnsp_gadget_exit(struct cdns * cdns)1946 static void cdnsp_gadget_exit(struct cdns *cdns)
1947 {
1948 	struct cdnsp_device *pdev = cdns->gadget_dev;
1949 
1950 	devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1951 	pm_runtime_mark_last_busy(cdns->dev);
1952 	pm_runtime_put_autosuspend(cdns->dev);
1953 	usb_del_gadget_udc(&pdev->gadget);
1954 	cdnsp_gadget_free_endpoints(pdev);
1955 	cdnsp_mem_cleanup(pdev);
1956 	kfree(pdev);
1957 	cdns->gadget_dev = NULL;
1958 	cdns_drd_gadget_off(cdns);
1959 }
1960 
cdnsp_gadget_suspend(struct cdns * cdns,bool do_wakeup)1961 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1962 {
1963 	struct cdnsp_device *pdev = cdns->gadget_dev;
1964 	unsigned long flags;
1965 
1966 	if (pdev->link_state == XDEV_U3)
1967 		return 0;
1968 
1969 	spin_lock_irqsave(&pdev->lock, flags);
1970 	cdnsp_disconnect_gadget(pdev);
1971 	cdnsp_stop(pdev);
1972 	spin_unlock_irqrestore(&pdev->lock, flags);
1973 
1974 	return 0;
1975 }
1976 
cdnsp_gadget_resume(struct cdns * cdns,bool hibernated)1977 static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1978 {
1979 	struct cdnsp_device *pdev = cdns->gadget_dev;
1980 	enum usb_device_speed max_speed;
1981 	unsigned long flags;
1982 	int ret;
1983 
1984 	if (!pdev->gadget_driver)
1985 		return 0;
1986 
1987 	spin_lock_irqsave(&pdev->lock, flags);
1988 	max_speed = pdev->gadget_driver->max_speed;
1989 
1990 	/* Limit speed if necessary. */
1991 	max_speed = min(max_speed, pdev->gadget.max_speed);
1992 
1993 	ret = cdnsp_run(pdev, max_speed);
1994 
1995 	if (pdev->link_state == XDEV_U3)
1996 		__cdnsp_gadget_wakeup(pdev);
1997 
1998 	spin_unlock_irqrestore(&pdev->lock, flags);
1999 
2000 	return ret;
2001 }
2002 
2003 /**
2004  * cdnsp_gadget_init - initialize device structure
2005  * @cdns: cdnsp instance
2006  *
2007  * This function initializes the gadget.
2008  */
cdnsp_gadget_init(struct cdns * cdns)2009 int cdnsp_gadget_init(struct cdns *cdns)
2010 {
2011 	struct cdns_role_driver *rdrv;
2012 
2013 	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2014 	if (!rdrv)
2015 		return -ENOMEM;
2016 
2017 	rdrv->start	= __cdnsp_gadget_init;
2018 	rdrv->stop	= cdnsp_gadget_exit;
2019 	rdrv->suspend	= cdnsp_gadget_suspend;
2020 	rdrv->resume	= cdnsp_gadget_resume;
2021 	rdrv->state	= CDNS_ROLE_STATE_INACTIVE;
2022 	rdrv->name	= "gadget";
2023 	cdns->roles[USB_ROLE_DEVICE] = rdrv;
2024 
2025 	return 0;
2026 }
2027