Lines Matching refs:dep
179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) in dwc3_ep_inc_enq() argument
181 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) in dwc3_ep_inc_deq() argument
190 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, in dwc3_gadget_del_and_unmap_request() argument
196 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
213 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, in dwc3_gadget_giveback() argument
230 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
232 dwc3_gadget_del_and_unmap_request(dep, req, status); in dwc3_gadget_giveback()
236 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
291 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, in dwc3_send_gadget_ep_cmd() argument
294 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
295 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
361 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
362 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
363 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
387 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
397 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
407 dep->name); in dwc3_send_gadget_ep_cmd()
438 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); in dwc3_send_gadget_ep_cmd()
442 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
445 dwc3_gadget_ep_get_transfer_index(dep); in dwc3_send_gadget_ep_cmd()
457 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) in dwc3_send_clear_stall_ep_cmd() argument
459 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
471 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
478 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_send_clear_stall_ep_cmd()
481 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, in dwc3_trb_dma_offset() argument
484 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
486 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
489 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) in dwc3_alloc_trb_pool() argument
491 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
493 if (dep->trb_pool) in dwc3_alloc_trb_pool()
496 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
498 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
499 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
500 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
501 dep->name); in dwc3_alloc_trb_pool()
508 static void dwc3_free_trb_pool(struct dwc3_ep *dep) in dwc3_free_trb_pool() argument
510 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
513 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
515 dep->trb_pool = NULL; in dwc3_free_trb_pool()
516 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
519 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) in dwc3_gadget_set_xfer_resource() argument
527 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, in dwc3_gadget_set_xfer_resource()
564 static int dwc3_gadget_start_config(struct dwc3_ep *dep) in dwc3_gadget_start_config() argument
572 if (dep->number) in dwc3_gadget_start_config()
577 dwc = dep->dwc; in dwc3_gadget_start_config()
579 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_gadget_start_config()
584 struct dwc3_ep *dep = dwc->eps[i]; in dwc3_gadget_start_config() local
586 if (!dep) in dwc3_gadget_start_config()
589 ret = dwc3_gadget_set_xfer_resource(dep); in dwc3_gadget_start_config()
597 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) in dwc3_gadget_set_ep_config() argument
602 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
604 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
605 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
614 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
621 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
626 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
633 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
645 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
651 if (dep->direction) in dwc3_gadget_set_ep_config()
652 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
669 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
671 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
676 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); in dwc3_gadget_set_ep_config()
723 struct dwc3_ep *dep; in dwc3_gadget_clear_tx_fifos() local
732 dep = dwc->eps[1]; in dwc3_gadget_clear_tx_fifos()
743 dep = dwc->eps[num]; in dwc3_gadget_clear_tx_fifos()
750 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_clear_tx_fifos()
776 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) in dwc3_gadget_resize_tx_fifos() argument
778 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_resize_tx_fifos()
793 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) in dwc3_gadget_resize_tx_fifos()
797 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) in dwc3_gadget_resize_tx_fifos()
802 if ((dep->endpoint.maxburst > 1 && in dwc3_gadget_resize_tx_fifos()
803 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || in dwc3_gadget_resize_tx_fifos()
804 usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
807 if (dep->endpoint.maxburst > 6 && in dwc3_gadget_resize_tx_fifos()
808 (usb_endpoint_xfer_bulk(dep->endpoint.desc) || in dwc3_gadget_resize_tx_fifos()
809 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) in dwc3_gadget_resize_tx_fifos()
850 dep->endpoint.name, fifo_size); in dwc3_gadget_resize_tx_fifos()
860 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in dwc3_gadget_resize_tx_fifos()
861 dep->flags |= DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_resize_tx_fifos()
875 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) in __dwc3_gadget_ep_enable() argument
877 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
878 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
883 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
884 ret = dwc3_gadget_resize_tx_fifos(dep); in __dwc3_gadget_ep_enable()
888 ret = dwc3_gadget_start_config(dep); in __dwc3_gadget_ep_enable()
893 ret = dwc3_gadget_set_ep_config(dep, action); in __dwc3_gadget_ep_enable()
897 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
901 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
902 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
905 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
908 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
909 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
915 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
919 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
921 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
922 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
923 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
940 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
941 trb_dma = dwc3_trb_dma_offset(dep, trb); in __dwc3_gadget_ep_enable()
948 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_gadget_ep_enable()
952 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
965 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_ep_enable()
977 if (!dep->direction || in __dwc3_gadget_ep_enable()
980 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
985 trace_dwc3_gadget_ep_enable(dep); in __dwc3_gadget_ep_enable()
990 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) in dwc3_remove_requests() argument
994 dwc3_stop_active_transfer(dep, true, false); in dwc3_remove_requests()
997 if (dep->flags & DWC3_EP_DELAY_STOP) in dwc3_remove_requests()
1001 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
1002 req = next_request(&dep->started_list); in dwc3_remove_requests()
1004 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1007 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
1008 req = next_request(&dep->pending_list); in dwc3_remove_requests()
1010 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1013 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
1014 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
1016 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1030 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) in __dwc3_gadget_ep_disable() argument
1032 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
1036 trace_dwc3_gadget_ep_disable(dep); in __dwc3_gadget_ep_disable()
1039 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
1040 __dwc3_gadget_ep_set_halt(dep, 0, false); in __dwc3_gadget_ep_disable()
1043 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
1046 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in __dwc3_gadget_ep_disable()
1048 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
1049 dep->type = 0; in __dwc3_gadget_ep_disable()
1056 if (dep->flags & DWC3_EP_DELAY_STOP) in __dwc3_gadget_ep_disable()
1058 dep->flags &= mask; in __dwc3_gadget_ep_disable()
1061 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
1062 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
1063 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
1087 struct dwc3_ep *dep; in dwc3_gadget_ep_enable() local
1102 dep = to_dwc3_ep(ep); in dwc3_gadget_ep_enable()
1103 dwc = dep->dwc; in dwc3_gadget_ep_enable()
1105 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
1107 dep->name)) in dwc3_gadget_ep_enable()
1111 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in dwc3_gadget_ep_enable()
1119 struct dwc3_ep *dep; in dwc3_gadget_ep_disable() local
1129 dep = to_dwc3_ep(ep); in dwc3_gadget_ep_disable()
1130 dwc = dep->dwc; in dwc3_gadget_ep_disable()
1132 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
1134 dep->name)) in dwc3_gadget_ep_disable()
1138 ret = __dwc3_gadget_ep_disable(dep); in dwc3_gadget_ep_disable()
1148 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_alloc_request() local
1154 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
1155 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
1156 req->dep = dep; in dwc3_gadget_ep_alloc_request()
1182 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) in dwc3_ep_prev_trb() argument
1189 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
1192 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) in dwc3_calc_trbs_left() argument
1201 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
1206 if (!list_empty(&dep->started_list)) in dwc3_calc_trbs_left()
1212 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
1215 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
1231 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, in dwc3_prepare_one_trb() argument
1242 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_one_trb()
1247 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1253 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1258 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1267 switch (usb_endpoint_type(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1299 struct usb_ep *ep = &dep->endpoint; in dwc3_prepare_one_trb()
1329 usb_endpoint_type(dep->endpoint.desc)); in dwc3_prepare_one_trb()
1336 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1337 if (!dep->stream_capable) in dwc3_prepare_one_trb()
1345 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_one_trb()
1353 else if (dep->stream_capable && is_last && in dwc3_prepare_one_trb()
1357 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in dwc3_prepare_one_trb()
1375 dwc3_ep_inc_enq(dep); in dwc3_prepare_one_trb()
1377 trace_dwc3_prepare_trb(dep, trb); in dwc3_prepare_one_trb()
1380 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) in dwc3_needs_extra_trb() argument
1382 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1386 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1402 static int dwc3_prepare_last_sg(struct dwc3_ep *dep, in dwc3_prepare_last_sg() argument
1406 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1410 if (dwc3_needs_extra_trb(dep, req)) in dwc3_prepare_last_sg()
1413 if (dwc3_calc_trbs_left(dep) < num_trbs) in dwc3_prepare_last_sg()
1420 dwc3_prepare_one_trb(dep, req, entry_length, in dwc3_prepare_last_sg()
1425 dwc3_prepare_one_trb(dep, req, in dwc3_prepare_last_sg()
1432 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, in dwc3_prepare_trbs_sg() argument
1442 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); in dwc3_prepare_trbs_sg()
1452 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); in dwc3_prepare_trbs_sg()
1475 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) in dwc3_prepare_trbs_sg()
1490 list_for_each_entry(r, &dep->started_list, list) { in dwc3_prepare_trbs_sg()
1499 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, in dwc3_prepare_trbs_sg()
1533 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, in dwc3_prepare_trbs_linear() argument
1536 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1549 static int dwc3_prepare_trbs(struct dwc3_ep *dep) in dwc3_prepare_trbs() argument
1566 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1568 ret = dwc3_prepare_trbs_sg(dep, req); in dwc3_prepare_trbs()
1573 if (!dwc3_calc_trbs_left(dep)) in dwc3_prepare_trbs()
1581 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1582 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in dwc3_prepare_trbs()
1586 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1587 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1590 dep->direction); in dwc3_prepare_trbs()
1600 ret = dwc3_prepare_trbs_sg(dep, req); in dwc3_prepare_trbs()
1604 ret = dwc3_prepare_trbs_linear(dep, req); in dwc3_prepare_trbs()
1607 if (!ret || !dwc3_calc_trbs_left(dep)) in dwc3_prepare_trbs()
1615 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1623 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1625 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) in __dwc3_gadget_kick_transfer() argument
1638 ret = dwc3_prepare_trbs(dep); in __dwc3_gadget_kick_transfer()
1642 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1651 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1653 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1664 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1667 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1668 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1671 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1674 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_gadget_kick_transfer()
1681 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_kick_transfer()
1683 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1687 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1688 dwc3_gadget_ep_cleanup_cancelled_requests(dep); in __dwc3_gadget_kick_transfer()
1693 if (dep->stream_capable && req->request.is_last && in __dwc3_gadget_kick_transfer()
1694 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in __dwc3_gadget_kick_transfer()
1695 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1719 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) in __dwc3_stop_active_transfer() argument
1721 struct dwc3 *dwc = dep->dwc; in __dwc3_stop_active_transfer()
1729 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_stop_active_transfer()
1731 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_stop_active_transfer()
1738 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { in __dwc3_stop_active_transfer()
1739 dep->flags |= DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1743 dep->resource_index = 0; in __dwc3_stop_active_transfer()
1748 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in __dwc3_stop_active_transfer()
1750 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_stop_active_transfer()
1753 dep->flags &= ~DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1799 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) in dwc3_gadget_start_isoc_quirk() argument
1805 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1814 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1815 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1816 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1818 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1819 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1823 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_gadget_start_isoc_quirk()
1827 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1828 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1833 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1834 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1836 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1843 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_start_isoc_quirk()
1849 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1853 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1855 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1857 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1859 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1861 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1862 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1863 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1866 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1867 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1869 return __dwc3_gadget_kick_transfer(dep); in dwc3_gadget_start_isoc_quirk()
1872 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) in __dwc3_gadget_start_isoc() argument
1874 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1875 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1879 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1880 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1881 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1888 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1889 return dwc3_gadget_start_isoc_quirk(dep); in __dwc3_gadget_start_isoc()
1896 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
1907 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
1910 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
1920 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); in __dwc3_gadget_start_isoc()
1922 ret = __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_start_isoc()
1933 ret = __dwc3_stop_active_transfer(dep, false, true); in __dwc3_gadget_start_isoc()
1938 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) in __dwc3_gadget_ep_queue() argument
1940 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
1942 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
1944 dep->name); in __dwc3_gadget_ep_queue()
1948 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", in __dwc3_gadget_ep_queue()
1949 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
1954 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
1964 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
1967 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
1974 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
1975 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
1976 (dep->flags & DWC3_EP_DELAY_STOP) || in __dwc3_gadget_ep_queue()
1977 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
1978 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
1990 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
1991 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { in __dwc3_gadget_ep_queue()
1992 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) in __dwc3_gadget_ep_queue()
1993 return __dwc3_gadget_start_isoc(dep); in __dwc3_gadget_ep_queue()
1999 __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_ep_queue()
2008 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_queue() local
2009 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
2016 ret = __dwc3_gadget_ep_queue(dep, req); in dwc3_gadget_ep_queue()
2022 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) in dwc3_gadget_ep_skip_trbs() argument
2043 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
2045 dwc3_ep_inc_deq(dep); in dwc3_gadget_ep_skip_trbs()
2051 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) in dwc3_gadget_ep_cleanup_cancelled_requests() argument
2054 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_cancelled_requests()
2056 while (!list_empty(&dep->cancelled_list)) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2057 req = next_request(&dep->cancelled_list); in dwc3_gadget_ep_cleanup_cancelled_requests()
2058 dwc3_gadget_ep_skip_trbs(dep, req); in dwc3_gadget_ep_cleanup_cancelled_requests()
2061 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_gadget_ep_cleanup_cancelled_requests()
2064 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2067 dwc3_gadget_giveback(dep, req, -EPIPE); in dwc3_gadget_ep_cleanup_cancelled_requests()
2071 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2078 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_cancelled_requests()
2089 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_dequeue() local
2090 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
2099 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
2104 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
2113 if (dep->number > 1) in dwc3_gadget_ep_dequeue()
2114 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
2121 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
2126 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_ep_dequeue()
2132 list_for_each_entry_safe(r, t, &dep->started_list, list) in dwc3_gadget_ep_dequeue()
2136 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
2151 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) in __dwc3_gadget_ep_set_halt() argument
2154 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
2159 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
2160 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
2172 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
2173 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
2175 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
2178 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
2180 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
2181 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
2185 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, in __dwc3_gadget_ep_set_halt()
2189 dep->name); in __dwc3_gadget_ep_set_halt()
2191 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
2198 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
2199 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2203 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_ep_set_halt()
2205 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_ep_set_halt()
2208 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || in __dwc3_gadget_ep_set_halt()
2209 (dep->flags & DWC3_EP_DELAY_STOP)) { in __dwc3_gadget_ep_set_halt()
2210 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
2212 dwc->clear_stall_protocol = dep->number; in __dwc3_gadget_ep_set_halt()
2217 dwc3_gadget_ep_cleanup_cancelled_requests(dep); in __dwc3_gadget_ep_set_halt()
2219 ret = dwc3_send_clear_stall_ep_cmd(dep); in __dwc3_gadget_ep_set_halt()
2222 dep->name); in __dwc3_gadget_ep_set_halt()
2226 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2228 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
2229 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
2230 __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_ep_set_halt()
2232 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2240 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_set_halt() local
2241 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
2248 ret = __dwc3_gadget_ep_set_halt(dep, value, false); in dwc3_gadget_ep_set_halt()
2256 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_set_wedge() local
2257 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
2262 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
2264 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
2267 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); in dwc3_gadget_ep_set_wedge()
2499 struct dwc3_ep *dep; in dwc3_stop_active_transfers() local
2501 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2502 if (!dep) in dwc3_stop_active_transfers()
2505 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in dwc3_stop_active_transfers()
2863 struct dwc3_ep *dep; in __dwc3_gadget_start() local
2916 dep = dwc->eps[0]; in __dwc3_gadget_start()
2917 dep->flags = 0; in __dwc3_gadget_start()
2918 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in __dwc3_gadget_start()
2920 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2924 dep = dwc->eps[1]; in __dwc3_gadget_start()
2925 dep->flags = 0; in __dwc3_gadget_start()
2926 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in __dwc3_gadget_start()
2928 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
3150 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_control_endpoint() argument
3152 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
3154 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
3155 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
3156 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
3157 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
3158 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
3160 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
3165 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_in_endpoint() argument
3167 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
3177 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
3201 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
3203 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
3204 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
3205 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
3207 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
3208 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
3209 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
3211 return dwc3_alloc_trb_pool(dep); in dwc3_gadget_init_in_endpoint()
3214 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_out_endpoint() argument
3216 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
3249 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
3250 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
3251 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
3252 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
3254 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
3255 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
3256 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
3258 return dwc3_alloc_trb_pool(dep); in dwc3_gadget_init_out_endpoint()
3263 struct dwc3_ep *dep; in dwc3_gadget_init_endpoint() local
3268 dep = kzalloc(sizeof(*dep), GFP_KERNEL); in dwc3_gadget_init_endpoint()
3269 if (!dep) in dwc3_gadget_init_endpoint()
3272 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
3273 dep->number = epnum; in dwc3_gadget_init_endpoint()
3274 dep->direction = direction; in dwc3_gadget_init_endpoint()
3275 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
3276 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
3277 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
3278 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
3280 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
3283 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
3285 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
3286 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
3287 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
3291 ret = dwc3_gadget_init_control_endpoint(dep); in dwc3_gadget_init_endpoint()
3293 ret = dwc3_gadget_init_in_endpoint(dep); in dwc3_gadget_init_endpoint()
3295 ret = dwc3_gadget_init_out_endpoint(dep); in dwc3_gadget_init_endpoint()
3300 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
3301 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
3303 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
3304 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
3305 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
3307 dwc3_debugfs_create_endpoint_dir(dep); in dwc3_gadget_init_endpoint()
3331 struct dwc3_ep *dep; in dwc3_gadget_free_endpoints() local
3335 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
3336 if (!dep) in dwc3_gadget_free_endpoints()
3348 dwc3_free_trb_pool(dep); in dwc3_gadget_free_endpoints()
3349 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
3352 dwc3_debugfs_remove_endpoint_dir(dep); in dwc3_gadget_free_endpoints()
3353 kfree(dep); in dwc3_gadget_free_endpoints()
3359 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_completed_trb() argument
3365 dwc3_ep_inc_deq(dep); in dwc3_gadget_ep_reclaim_completed_trb()
3367 trace_dwc3_complete_trb(dep, trb); in dwc3_gadget_ep_reclaim_completed_trb()
3387 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
3392 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
3401 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
3402 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
3427 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_trb_sg() argument
3431 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3439 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3444 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, in dwc3_gadget_ep_reclaim_trb_sg()
3453 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_trb_linear() argument
3457 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_linear()
3459 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, in dwc3_gadget_ep_reclaim_trb_linear()
3468 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, in dwc3_gadget_ep_cleanup_completed_request() argument
3476 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3479 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3488 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3502 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); in dwc3_gadget_ep_cleanup_completed_request()
3521 dwc3_gadget_giveback(dep, req, request_status); in dwc3_gadget_ep_cleanup_completed_request()
3527 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, in dwc3_gadget_ep_cleanup_completed_requests() argument
3532 while (!list_empty(&dep->started_list)) { in dwc3_gadget_ep_cleanup_completed_requests()
3535 req = next_request(&dep->started_list); in dwc3_gadget_ep_cleanup_completed_requests()
3536 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, in dwc3_gadget_ep_cleanup_completed_requests()
3544 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_completed_requests()
3549 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) in dwc3_gadget_ep_should_continue() argument
3552 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_should_continue()
3554 if (!dep->endpoint.desc || !dwc->pullups_connected || in dwc3_gadget_ep_should_continue()
3558 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
3565 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
3572 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, in dwc3_gadget_endpoint_frame_from_event() argument
3575 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
3578 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_trbs_complete() argument
3581 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
3584 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); in dwc3_gadget_endpoint_trbs_complete()
3586 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
3589 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_trbs_complete()
3592 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
3593 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
3594 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3595 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_endpoint_trbs_complete()
3596 else if (dwc3_gadget_ep_should_continue(dep)) in dwc3_gadget_endpoint_trbs_complete()
3597 if (__dwc3_gadget_kick_transfer(dep) == 0) in dwc3_gadget_endpoint_trbs_complete()
3610 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3612 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3615 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3629 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_in_progress() argument
3634 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_transfer_in_progress()
3637 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3638 dwc3_gadget_endpoint_frame_from_event(dep, event); in dwc3_gadget_endpoint_transfer_in_progress()
3646 dwc3_gadget_endpoint_trbs_complete(dep, event, status); in dwc3_gadget_endpoint_transfer_in_progress()
3649 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_complete() argument
3654 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3659 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) in dwc3_gadget_endpoint_transfer_complete()
3660 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3663 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_not_ready() argument
3666 dwc3_gadget_endpoint_frame_from_event(dep, event); in dwc3_gadget_endpoint_transfer_not_ready()
3676 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3679 (void) __dwc3_gadget_start_isoc(dep); in dwc3_gadget_endpoint_transfer_not_ready()
3682 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_command_complete() argument
3695 if (dep->stream_capable) in dwc3_gadget_endpoint_command_complete()
3696 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_command_complete()
3698 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3699 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3700 dwc3_gadget_ep_cleanup_cancelled_requests(dep); in dwc3_gadget_endpoint_command_complete()
3702 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3703 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3705 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3706 if (dwc3_send_clear_stall_ep_cmd(dep)) { in dwc3_gadget_endpoint_command_complete()
3709 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3715 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3716 if (dwc->clear_stall_protocol == dep->number) in dwc3_gadget_endpoint_command_complete()
3720 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3721 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3722 __dwc3_gadget_kick_transfer(dep); in dwc3_gadget_endpoint_command_complete()
3724 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3727 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, in dwc3_gadget_endpoint_stream_event() argument
3730 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_stream_event()
3733 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3747 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { in dwc3_gadget_endpoint_stream_event()
3748 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) in dwc3_gadget_endpoint_stream_event()
3749 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; in dwc3_gadget_endpoint_stream_event()
3751 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3756 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_gadget_endpoint_stream_event()
3757 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || in dwc3_gadget_endpoint_stream_event()
3759 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) in dwc3_gadget_endpoint_stream_event()
3779 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_gadget_endpoint_stream_event()
3781 dep->flags |= DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_stream_event()
3782 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_endpoint_stream_event()
3789 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3795 struct dwc3_ep *dep; in dwc3_endpoint_interrupt() local
3798 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3800 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3801 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3817 dwc3_gadget_endpoint_transfer_in_progress(dep, event); in dwc3_endpoint_interrupt()
3820 dwc3_gadget_endpoint_transfer_not_ready(dep, event); in dwc3_endpoint_interrupt()
3823 dwc3_gadget_endpoint_command_complete(dep, event); in dwc3_endpoint_interrupt()
3826 dwc3_gadget_endpoint_transfer_complete(dep, event); in dwc3_endpoint_interrupt()
3829 dwc3_gadget_endpoint_stream_event(dep, event); in dwc3_endpoint_interrupt()
3878 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, in dwc3_stop_active_transfer() argument
3881 struct dwc3 *dwc = dep->dwc; in dwc3_stop_active_transfer()
3889 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) in dwc3_stop_active_transfer()
3892 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) in dwc3_stop_active_transfer()
3895 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3896 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
3907 dep->flags |= DWC3_EP_DELAY_STOP; in dwc3_stop_active_transfer()
3942 __dwc3_stop_active_transfer(dep, force, interrupt); in dwc3_stop_active_transfer()
3950 struct dwc3_ep *dep; in dwc3_clear_stall_all_ep() local
3953 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
3954 if (!dep) in dwc3_clear_stall_all_ep()
3957 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
3960 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
3962 ret = dwc3_send_clear_stall_ep_cmd(dep); in dwc3_clear_stall_all_ep()
4080 struct dwc3_ep *dep; in dwc3_gadget_conndone_interrupt() local
4199 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
4200 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); in dwc3_gadget_conndone_interrupt()
4202 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4206 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
4207 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); in dwc3_gadget_conndone_interrupt()
4209 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()