114 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,  in cdns3_get_dma_pos()  argument
119 	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;  in cdns3_get_dma_pos()
162 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)  in cdns3_select_ep()  argument
164 	if (priv_dev->selected_ep == ep)  in cdns3_select_ep()
167 	priv_dev->selected_ep = ep;  in cdns3_select_ep()
168 	writel(ep, &priv_dev->regs->ep_sel);  in cdns3_select_ep()
178 static int cdns3_get_tdl(struct cdns3_device *priv_dev)  in cdns3_get_tdl()  argument
180 	if (priv_dev->dev_ver < DEV_VER_V3)  in cdns3_get_tdl()
181 		return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_get_tdl()
183 		return readl(&priv_dev->regs->ep_tdl);  in cdns3_get_tdl()
196 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_free_trb_pool()  local
199 		dma_pool_free(priv_dev->eps_dma_pool,  in cdns3_free_trb_pool()
213 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_allocate_trb_pool()  local
222 		priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,  in cdns3_allocate_trb_pool()
263 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_stall_flush()  local
269 	       &priv_dev->regs->ep_cmd);  in cdns3_ep_stall_flush()
272 	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_ep_stall_flush()
282 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)  in cdns3_hw_reset_eps_config()  argument
286 	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);  in cdns3_hw_reset_eps_config()
288 	cdns3_allow_enable_l1(priv_dev, 0);  in cdns3_hw_reset_eps_config()
289 	priv_dev->hw_configured_flag = 0;  in cdns3_hw_reset_eps_config()
290 	priv_dev->onchip_used_size = 0;  in cdns3_hw_reset_eps_config()
291 	priv_dev->out_mem_is_allocated = 0;  in cdns3_hw_reset_eps_config()
292 	priv_dev->wait_for_setup = 0;  in cdns3_hw_reset_eps_config()
293 	priv_dev->using_streams = 0;  in cdns3_hw_reset_eps_config()
296 		if (priv_dev->eps[i])  in cdns3_hw_reset_eps_config()
297 			priv_dev->eps[i]->flags &= ~EP_CONFIGURED;  in cdns3_hw_reset_eps_config()
350 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)  in cdns3_allow_enable_l1()  argument
353 		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);  in cdns3_allow_enable_l1()
355 		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);  in cdns3_allow_enable_l1()
358 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)  in cdns3_get_speed()  argument
362 	reg = readl(&priv_dev->regs->usb_sts);  in cdns3_get_speed()
383 static int cdns3_start_all_request(struct cdns3_device *priv_dev,  in cdns3_start_all_request()  argument
403 			dev_dbg(priv_dev->dev, "Blocking external request\n");  in cdns3_start_all_request()
435 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \  argument
507 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,  in cdns3_wa2_gadget_giveback()  argument
523 		usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,  in cdns3_wa2_gadget_giveback()
531 			usb_gadget_map_request_by_dev(priv_dev->sysdev, req,  in cdns3_wa2_gadget_giveback()
540 		cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_wa2_gadget_giveback()
547 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,  in cdns3_wa2_gadget_ep_queue()  argument
561 		cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);  in cdns3_wa2_gadget_ep_queue()
563 		reg = readl(&priv_dev->regs->ep_sts_en);  in cdns3_wa2_gadget_ep_queue()
566 		writel(reg, &priv_dev->regs->ep_sts_en);  in cdns3_wa2_gadget_ep_queue()
709 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)  in cdns3_wa2_reset_tdl()  argument
711 	u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_wa2_reset_tdl()
717 		       &priv_dev->regs->ep_cmd);  in cdns3_wa2_reset_tdl()
721 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)  in cdns3_wa2_check_outq_status()  argument
726 	cdns3_select_ep(priv_dev, 0);  in cdns3_wa2_check_outq_status()
728 	ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in cdns3_wa2_check_outq_status()
732 		struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];  in cdns3_wa2_check_outq_status()
745 				cdns3_select_ep(priv_dev, outq_ep->num |  in cdns3_wa2_check_outq_status()
747 				ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);  in cdns3_wa2_check_outq_status()
748 				ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);  in cdns3_wa2_check_outq_status()
751 				cdns3_set_register_bit(&priv_dev->regs->ep_cfg,  in cdns3_wa2_check_outq_status()
754 				cdns3_wa2_enable_detection(priv_dev, outq_ep,  in cdns3_wa2_check_outq_status()
757 				       &priv_dev->regs->ep_sts_en);  in cdns3_wa2_check_outq_status()
759 				cdns3_wa2_reset_tdl(priv_dev);  in cdns3_wa2_check_outq_status()
774 					       &priv_dev->regs->ep_cmd);  in cdns3_wa2_check_outq_status()
795 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_giveback()  local
803 	usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,  in cdns3_gadget_giveback()
809 		dma_sync_single_for_cpu(priv_dev->sysdev,  in cdns3_gadget_giveback()
822 	if (priv_dev->dev_ver < DEV_VER_V2) {  in cdns3_gadget_giveback()
823 		request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,  in cdns3_gadget_giveback()
830 		spin_unlock(&priv_dev->lock);  in cdns3_gadget_giveback()
833 		spin_lock(&priv_dev->lock);  in cdns3_gadget_giveback()
836 	if (request->buf == priv_dev->zlp_buf)  in cdns3_gadget_giveback()
860 	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,  in cdns3_free_aligned_request_buf()  local
865 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
867 	list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {  in cdns3_free_aligned_request_buf()
876 			spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
877 			dma_free_noncoherent(priv_dev->sysdev, buf->size,  in cdns3_free_aligned_request_buf()
880 			spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
884 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
890 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_prepare_aligned_request_buf()  local
908 		buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
922 				   &priv_dev->aligned_buf_wq);  in cdns3_prepare_aligned_request_buf()
929 			      &priv_dev->aligned_buf_list);  in cdns3_prepare_aligned_request_buf()
934 		dma_sync_single_for_cpu(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
941 	dma_sync_single_for_device(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
953 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_wa1_update_guard()  local
958 		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_wa1_update_guard()
972 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,  in cdns3_wa1_tray_restore_cycle_bit()  argument
978 	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_wa1_tray_restore_cycle_bit()
979 	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_wa1_tray_restore_cycle_bit()
988 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_run_stream_transfer()  local
1037 	if (priv_dev->dev_ver >= DEV_VER_V2) {  in cdns3_ep_run_stream_transfer()
1038 		if (priv_dev->gadget.speed == USB_SPEED_SUPER)  in cdns3_ep_run_stream_transfer()
1055 	       &priv_dev->regs->ep_traddr);  in cdns3_ep_run_stream_transfer()
1060 		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);  in cdns3_ep_run_stream_transfer()
1069 		if (priv_dev->dev_ver < DEV_VER_V2)  in cdns3_ep_run_stream_transfer()
1071 			       &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1072 		else if (priv_dev->dev_ver > DEV_VER_V2)  in cdns3_ep_run_stream_transfer()
1073 			writel(tdl, &priv_dev->regs->ep_tdl);  in cdns3_ep_run_stream_transfer()
1076 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1078 		       EP_CMD_ERDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1081 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_ep_run_stream_transfer()
1085 	__cdns3_gadget_wakeup(priv_dev);  in cdns3_ep_run_stream_transfer()
1092 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_rearm_drdy_if_needed()  local
1094 	if (priv_dev->dev_ver < DEV_VER_V3)  in cdns3_rearm_drdy_if_needed()
1097 	if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {  in cdns3_rearm_drdy_if_needed()
1098 		writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);  in cdns3_rearm_drdy_if_needed()
1099 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_rearm_drdy_if_needed()
1113 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_run_transfer()  local
1160 		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_ep_run_transfer()
1161 		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_ep_run_transfer()
1187 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_ep_run_transfer()
1196 	if (priv_dev->dev_ver >= DEV_VER_V2) {  in cdns3_ep_run_transfer()
1201 		if (priv_dev->gadget.speed == USB_SPEED_SUPER)  in cdns3_ep_run_transfer()
1273 	if (priv_dev->dev_ver < DEV_VER_V2 &&  in cdns3_ep_run_transfer()
1276 		u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_ep_run_transfer()
1286 			       &priv_dev->regs->ep_cmd);  in cdns3_ep_run_transfer()
1299 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_ep_run_transfer()
1300 		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);  in cdns3_ep_run_transfer()
1338 			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,  in cdns3_ep_run_transfer()
1344 					&priv_dev->regs->ep_traddr);  in cdns3_ep_run_transfer()
1352 		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);  in cdns3_ep_run_transfer()
1353 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_transfer()
1356 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_ep_run_transfer()
1360 	__cdns3_gadget_wakeup(priv_dev);  in cdns3_ep_run_transfer()
1365 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)  in cdns3_set_hw_configuration()  argument
1370 	if (priv_dev->hw_configured_flag)  in cdns3_set_hw_configuration()
1373 	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);  in cdns3_set_hw_configuration()
1375 	cdns3_set_register_bit(&priv_dev->regs->usb_conf,  in cdns3_set_hw_configuration()
1378 	priv_dev->hw_configured_flag = 1;  in cdns3_set_hw_configuration()
1380 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_set_hw_configuration()
1383 			cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_set_hw_configuration()
1387 	cdns3_allow_enable_l1(priv_dev, 1);  in cdns3_set_hw_configuration()
1432 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_trb_handled()  local
1438 	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_trb_handled()
1439 	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_trb_handled()
1492 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,  in cdns3_transfer_completed()  argument
1518 			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_transfer_completed()
1555 			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_transfer_completed()
1561 				dev_warn(priv_dev->dev,  in cdns3_transfer_completed()
1583 		cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_transfer_completed()
1588 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_rearm_transfer()  local
1597 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_rearm_transfer()
1599 		__cdns3_gadget_wakeup(priv_dev);  in cdns3_rearm_transfer()
1602 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_rearm_transfer()
1609 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_reprogram_tdl()  local
1618 	writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);  in cdns3_reprogram_tdl()
1629 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_check_ep_interrupt_proceed()  local
1635 	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_check_ep_interrupt_proceed()
1637 	trace_cdns3_epx_irq(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1639 	ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in cdns3_check_ep_interrupt_proceed()
1640 	writel(ep_sts_reg, &priv_dev->regs->ep_sts);  in cdns3_check_ep_interrupt_proceed()
1645 		tdl = cdns3_get_tdl(priv_dev);  in cdns3_check_ep_interrupt_proceed()
1658 			       &priv_dev->regs->ep_cmd);  in cdns3_check_ep_interrupt_proceed()
1667 				cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1675 		    priv_dev->dev_ver < DEV_VER_V2)) {  in cdns3_check_ep_interrupt_proceed()
1689 				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);  in cdns3_check_ep_interrupt_proceed()
1692 				writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_check_ep_interrupt_proceed()
1696 			cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1701 				cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1721 				cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1730 			cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1741 		cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1750 	if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&  in cdns3_check_ep_interrupt_proceed()
1757 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)  in cdns3_disconnect_gadget()  argument
1759 	if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)  in cdns3_disconnect_gadget()
1760 		priv_dev->gadget_driver->disconnect(&priv_dev->gadget);  in cdns3_disconnect_gadget()
1769 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,  in cdns3_check_usb_interrupt_proceed()  argument
1771 __must_hold(&priv_dev->lock)  in cdns3_check_usb_interrupt_proceed()
1775 	trace_cdns3_usb_irq(priv_dev, usb_ists);  in cdns3_check_usb_interrupt_proceed()
1782 		if (readl(&priv_dev->regs->drbl))  in cdns3_check_usb_interrupt_proceed()
1783 			__cdns3_gadget_wakeup(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1788 		speed = cdns3_get_speed(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1789 		priv_dev->gadget.speed = speed;  in cdns3_check_usb_interrupt_proceed()
1790 		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);  in cdns3_check_usb_interrupt_proceed()
1791 		cdns3_ep0_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1796 		spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1797 		cdns3_disconnect_gadget(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1798 		spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1799 		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_check_usb_interrupt_proceed()
1800 		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);  in cdns3_check_usb_interrupt_proceed()
1801 		cdns3_hw_reset_eps_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1805 		if (priv_dev->gadget_driver &&  in cdns3_check_usb_interrupt_proceed()
1806 		    priv_dev->gadget_driver->suspend) {  in cdns3_check_usb_interrupt_proceed()
1807 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1808 			priv_dev->gadget_driver->suspend(&priv_dev->gadget);  in cdns3_check_usb_interrupt_proceed()
1809 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1814 		if (priv_dev->gadget_driver &&  in cdns3_check_usb_interrupt_proceed()
1815 		    priv_dev->gadget_driver->resume) {  in cdns3_check_usb_interrupt_proceed()
1816 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1817 			priv_dev->gadget_driver->resume(&priv_dev->gadget);  in cdns3_check_usb_interrupt_proceed()
1818 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1824 		if (priv_dev->gadget_driver) {  in cdns3_check_usb_interrupt_proceed()
1825 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1826 			usb_gadget_udc_reset(&priv_dev->gadget,  in cdns3_check_usb_interrupt_proceed()
1827 					     priv_dev->gadget_driver);  in cdns3_check_usb_interrupt_proceed()
1828 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1831 			speed = cdns3_get_speed(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1832 			priv_dev->gadget.speed = speed;  in cdns3_check_usb_interrupt_proceed()
1833 			cdns3_hw_reset_eps_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1834 			cdns3_ep0_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1849 	struct cdns3_device *priv_dev = data;  in cdns3_device_irq_handler()  local
1850 	struct cdns *cdns = dev_get_drvdata(priv_dev->dev);  in cdns3_device_irq_handler()
1858 	reg = readl(&priv_dev->regs->usb_ists);  in cdns3_device_irq_handler()
1867 		reg = ~reg & readl(&priv_dev->regs->usb_ien);  in cdns3_device_irq_handler()
1869 		writel(reg, &priv_dev->regs->usb_ien);  in cdns3_device_irq_handler()
1874 	reg = readl(&priv_dev->regs->ep_ists);  in cdns3_device_irq_handler()
1876 		writel(0, &priv_dev->regs->ep_ien);  in cdns3_device_irq_handler()
1894 	struct cdns3_device *priv_dev = data;  in cdns3_device_thread_irq_handler()  local
1900 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_device_thread_irq_handler()
1902 	reg = readl(&priv_dev->regs->usb_ists);  in cdns3_device_thread_irq_handler()
1904 		writel(reg, &priv_dev->regs->usb_ists);  in cdns3_device_thread_irq_handler()
1905 		writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);  in cdns3_device_thread_irq_handler()
1906 		cdns3_check_usb_interrupt_proceed(priv_dev, reg);  in cdns3_device_thread_irq_handler()
1910 	reg = readl(&priv_dev->regs->ep_ists);  in cdns3_device_thread_irq_handler()
1914 		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);  in cdns3_device_thread_irq_handler()
1920 		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);  in cdns3_device_thread_irq_handler()
1931 		cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);  in cdns3_device_thread_irq_handler()
1935 	if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)  in cdns3_device_thread_irq_handler()
1936 		cdns3_wa2_check_outq_status(priv_dev);  in cdns3_device_thread_irq_handler()
1939 	writel(~0, &priv_dev->regs->ep_ien);  in cdns3_device_thread_irq_handler()
1940 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_device_thread_irq_handler()
1957 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,  in cdns3_ep_onchip_buffer_reserve()  argument
1963 	remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;  in cdns3_ep_onchip_buffer_reserve()
1969 		priv_dev->onchip_used_size += size;  in cdns3_ep_onchip_buffer_reserve()
1977 		if (priv_dev->out_mem_is_allocated >= size)  in cdns3_ep_onchip_buffer_reserve()
1980 		required = size - priv_dev->out_mem_is_allocated;  in cdns3_ep_onchip_buffer_reserve()
1985 		priv_dev->out_mem_is_allocated += required;  in cdns3_ep_onchip_buffer_reserve()
1986 		priv_dev->onchip_used_size += required;  in cdns3_ep_onchip_buffer_reserve()
1992 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,  in cdns3_configure_dmult()  argument
1995 	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;  in cdns3_configure_dmult()
1998 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_configure_dmult()
2001 	if (priv_dev->dev_ver == DEV_VER_V2)  in cdns3_configure_dmult()
2004 	if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {  in cdns3_configure_dmult()
2034 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_config()  local
2043 	buffering = priv_dev->ep_buf_size - 1;  in cdns3_ep_config()
2045 	cdns3_configure_dmult(priv_dev, priv_ep);  in cdns3_ep_config()
2051 		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)  in cdns3_ep_config()
2057 		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)  in cdns3_ep_config()
2062 		mult = priv_dev->ep_iso_burst - 1;  in cdns3_ep_config()
2066 	switch (priv_dev->gadget.speed) {  in cdns3_ep_config()
2078 			maxburst = priv_dev->ep_iso_burst - 1;  in cdns3_ep_config()
2085 			maxburst = priv_dev->ep_buf_size - 1;  in cdns3_ep_config()
2105 	if (!priv_dev->hw_configured_flag) {  in cdns3_ep_config()
2106 		ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,  in cdns3_ep_config()
2109 			dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");  in cdns3_ep_config()
2117 	if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {  in cdns3_ep_config()
2118 		if (priv_dev->dev_ver >= DEV_VER_V3) {  in cdns3_ep_config()
2125 			cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,  in cdns3_ep_config()
2138 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_ep_config()
2139 	writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_ep_config()
2142 	dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",  in cdns3_ep_config()
2157 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,  in cdns3_find_available_ep()  argument
2163 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_find_available_ep()
2206 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_match_ep()  local
2210 	priv_ep = cdns3_find_available_ep(priv_dev, desc);  in cdns3_gadget_match_ep()
2212 		dev_err(priv_dev->dev, "no available ep\n");  in cdns3_gadget_match_ep()
2216 	dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);  in cdns3_gadget_match_ep()
2218 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_match_ep()
2225 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_match_ep()
2280 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_enable()  local
2295 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_enable()
2299 		dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");  in cdns3_gadget_ep_enable()
2304 		dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");  in cdns3_gadget_ep_enable()
2308 	if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,  in cdns3_gadget_ep_enable()
2312 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_enable()
2320 		dev_err(priv_dev->dev, "Driver is limited to %d period\n",  in cdns3_gadget_ep_enable()
2328 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_ep_enable()
2346 		if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {  in cdns3_gadget_ep_enable()
2352 			priv_dev->using_streams |= true;  in cdns3_gadget_ep_enable()
2366 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_ep_enable()
2370 	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_ep_enable()
2372 	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_ep_enable()
2383 	cdns3_set_register_bit(&priv_dev->regs->ep_ien,  in cdns3_gadget_ep_enable()
2386 	if (priv_dev->dev_ver < DEV_VER_V2)  in cdns3_gadget_ep_enable()
2387 		cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);  in cdns3_gadget_ep_enable()
2389 	writel(reg, &priv_dev->regs->ep_sts_en);  in cdns3_gadget_ep_enable()
2398 	reg = readl(&priv_dev->regs->ep_sts);  in cdns3_gadget_ep_enable()
2404 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_enable()
2419 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_disable()  local
2432 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_disable()
2434 	if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),  in cdns3_gadget_ep_disable()
2438 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_disable()
2442 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_disable()
2444 	ep_cfg = readl(&priv_dev->regs->ep_cfg);  in cdns3_gadget_ep_disable()
2446 	writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_gadget_ep_disable()
2453 	readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,  in cdns3_gadget_ep_disable()
2455 	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_ep_disable()
2457 	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_ep_disable()
2461 		dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",  in cdns3_gadget_ep_disable()
2494 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_disable()
2512 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_queue()  local
2521 	if (priv_dev->dev_ver < DEV_VER_V2) {  in __cdns3_gadget_ep_queue()
2522 		ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,  in __cdns3_gadget_ep_queue()
2533 	ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,  in __cdns3_gadget_ep_queue()
2548 		if (priv_dev->hw_configured_flag &&  in __cdns3_gadget_ep_queue()
2551 			cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_queue()
2553 		if (priv_dev->hw_configured_flag && priv_ep->prime_flag)  in __cdns3_gadget_ep_queue()
2554 			cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_queue()
2565 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_queue()  local
2573 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_queue()
2575 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_queue()
2584 		zlp_request->buf = priv_dev->zlp_buf;  in cdns3_gadget_ep_queue()
2590 		dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",  in cdns3_gadget_ep_queue()
2595 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_queue()
2610 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_dequeue()  local
2621 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_dequeue()
2623 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_dequeue()
2629 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_dequeue()
2664 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_dequeue()
2675 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_set_halt()  local
2680 		u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in __cdns3_gadget_ep_set_halt()
2696 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_clear_halt()  local
2716 	writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in __cdns3_gadget_ep_clear_halt()
2719 	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in __cdns3_gadget_ep_clear_halt()
2733 	cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_clear_halt()
2747 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_set_halt()  local
2754 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_set_halt()
2756 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_set_halt()
2765 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_set_halt()
2791 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_get_frame()  local
2793 	return readl(&priv_dev->regs->usb_itpn);  in cdns3_gadget_get_frame()
2796 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)  in __cdns3_gadget_wakeup()  argument
2800 	speed = cdns3_get_speed(priv_dev);  in __cdns3_gadget_wakeup()
2806 	writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);  in __cdns3_gadget_wakeup()
2813 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_wakeup()  local
2817 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_wakeup()
2818 	ret = __cdns3_gadget_wakeup(priv_dev);  in cdns3_gadget_wakeup()
2819 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_wakeup()
2826 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_set_selfpowered()  local
2829 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_set_selfpowered()
2830 	priv_dev->is_selfpowered = !!is_selfpowered;  in cdns3_gadget_set_selfpowered()
2831 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_set_selfpowered()
2837 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_pullup()  local
2840 		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);  in cdns3_gadget_pullup()
2842 		writel(~0, &priv_dev->regs->ep_ists);  in cdns3_gadget_pullup()
2843 		writel(~0, &priv_dev->regs->usb_ists);  in cdns3_gadget_pullup()
2844 		writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);  in cdns3_gadget_pullup()
2850 static void cdns3_gadget_config(struct cdns3_device *priv_dev)  in cdns3_gadget_config()  argument
2852 	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;  in cdns3_gadget_config()
2855 	cdns3_ep0_config(priv_dev);  in cdns3_gadget_config()
2864 	if (priv_dev->dev_ver == DEV_VER_TI_V1) {  in cdns3_gadget_config()
2887 	writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);  in cdns3_gadget_config()
2889 	cdns3_configure_dmult(priv_dev, NULL);  in cdns3_gadget_config()
2902 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_udc_start()  local
2906 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_udc_start()
2907 	priv_dev->gadget_driver = driver;  in cdns3_gadget_udc_start()
2914 		writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
2915 		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
2918 		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
2923 		dev_err(priv_dev->dev,  in cdns3_gadget_udc_start()
2933 	cdns3_gadget_config(priv_dev);  in cdns3_gadget_udc_start()
2934 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_udc_start()
2946 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_udc_stop()  local
2952 	priv_dev->gadget_driver = NULL;  in cdns3_gadget_udc_stop()
2954 	priv_dev->onchip_used_size = 0;  in cdns3_gadget_udc_stop()
2955 	priv_dev->out_mem_is_allocated = 0;  in cdns3_gadget_udc_stop()
2956 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_udc_stop()
2958 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_gadget_udc_stop()
2961 		cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_udc_stop()
2962 		writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_udc_stop()
2963 		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_udc_stop()
2970 	writel(0, &priv_dev->regs->usb_ien);  in cdns3_gadget_udc_stop()
2971 	writel(0, &priv_dev->regs->usb_pwr);  in cdns3_gadget_udc_stop()
2972 	writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_stop()
2989 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_check_config()  local
3002 	if (total > priv_dev->onchip_buffers)  in cdns3_gadget_check_config()
3005 	priv_dev->ep_buf_size = priv_dev->ep_iso_burst =  in cdns3_gadget_check_config()
3006 			(priv_dev->onchip_buffers - 2) / (n_in + 1);  in cdns3_gadget_check_config()
3022 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)  in cdns3_free_all_eps()  argument
3027 	priv_dev->eps[16] = NULL;  in cdns3_free_all_eps()
3030 		if (priv_dev->eps[i]) {  in cdns3_free_all_eps()
3031 			cdns3_free_trb_pool(priv_dev->eps[i]);  in cdns3_free_all_eps()
3032 			devm_kfree(priv_dev->dev, priv_dev->eps[i]);  in cdns3_free_all_eps()
3042 static int cdns3_init_eps(struct cdns3_device *priv_dev)  in cdns3_init_eps()  argument
3052 	ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);  in cdns3_init_eps()
3053 	iso_ep_reg = readl(&priv_dev->regs->usb_cap4);  in cdns3_init_eps()
3055 	dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");  in cdns3_init_eps()
3066 			priv_dev->eps[i] = priv_dev->eps[0];  in cdns3_init_eps()
3070 		priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),  in cdns3_init_eps()
3076 		priv_ep->cdns3_dev = priv_dev;  in cdns3_init_eps()
3077 		priv_dev->eps[i] = priv_ep;  in cdns3_init_eps()
3082 			ret = cdns3_init_ep0(priv_dev, priv_ep);  in cdns3_init_eps()
3084 				dev_err(priv_dev->dev, "Failed to init ep0\n");  in cdns3_init_eps()
3108 				      &priv_dev->gadget.ep_list);  in cdns3_init_eps()
3113 		dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",  in cdns3_init_eps()
3125 	cdns3_free_all_eps(priv_dev);  in cdns3_init_eps()
3131 	struct cdns3_device *priv_dev = container_of(dev,  in cdns3_gadget_release()  local
3134 	kfree(priv_dev);  in cdns3_gadget_release()
3139 	struct cdns3_device *priv_dev;  in cdns3_gadget_exit()  local
3141 	priv_dev = cdns->gadget_dev;  in cdns3_gadget_exit()
3147 	usb_del_gadget(&priv_dev->gadget);  in cdns3_gadget_exit()
3148 	devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);  in cdns3_gadget_exit()
3150 	cdns3_free_all_eps(priv_dev);  in cdns3_gadget_exit()
3152 	while (!list_empty(&priv_dev->aligned_buf_list)) {  in cdns3_gadget_exit()
3155 		buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);  in cdns3_gadget_exit()
3156 		dma_free_noncoherent(priv_dev->sysdev, buf->size,  in cdns3_gadget_exit()
3165 	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,  in cdns3_gadget_exit()
3166 			  priv_dev->setup_dma);  in cdns3_gadget_exit()
3167 	dma_pool_destroy(priv_dev->eps_dma_pool);  in cdns3_gadget_exit()
3169 	kfree(priv_dev->zlp_buf);  in cdns3_gadget_exit()
3170 	usb_put_gadget(&priv_dev->gadget);  in cdns3_gadget_exit()
3177 	struct cdns3_device *priv_dev;  in cdns3_gadget_start()  local
3181 	priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);  in cdns3_gadget_start()
3182 	if (!priv_dev)  in cdns3_gadget_start()
3185 	usb_initialize_gadget(cdns->dev, &priv_dev->gadget,  in cdns3_gadget_start()
3187 	cdns->gadget_dev = priv_dev;  in cdns3_gadget_start()
3188 	priv_dev->sysdev = cdns->dev;  in cdns3_gadget_start()
3189 	priv_dev->dev = cdns->dev;  in cdns3_gadget_start()
3190 	priv_dev->regs = cdns->dev_regs;  in cdns3_gadget_start()
3192 	device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",  in cdns3_gadget_start()
3193 				 &priv_dev->onchip_buffers);  in cdns3_gadget_start()
3195 	if (priv_dev->onchip_buffers <=  0) {  in cdns3_gadget_start()
3196 		u32 reg = readl(&priv_dev->regs->usb_cap2);  in cdns3_gadget_start()
3198 		priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);  in cdns3_gadget_start()
3201 	if (!priv_dev->onchip_buffers)  in cdns3_gadget_start()
3202 		priv_dev->onchip_buffers = 256;  in cdns3_gadget_start()
3223 	priv_dev->gadget.max_speed = max_speed;  in cdns3_gadget_start()
3224 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_start()
3225 	priv_dev->gadget.ops = &cdns3_gadget_ops;  in cdns3_gadget_start()
3226 	priv_dev->gadget.name = "usb-ss-gadget";  in cdns3_gadget_start()
3227 	priv_dev->gadget.quirk_avoids_skb_reserve = 1;  in cdns3_gadget_start()
3228 	priv_dev->gadget.irq = cdns->dev_irq;  in cdns3_gadget_start()
3230 	spin_lock_init(&priv_dev->lock);  in cdns3_gadget_start()
3231 	INIT_WORK(&priv_dev->pending_status_wq,  in cdns3_gadget_start()
3234 	INIT_WORK(&priv_dev->aligned_buf_wq,  in cdns3_gadget_start()
3238 	INIT_LIST_HEAD(&priv_dev->gadget.ep_list);  in cdns3_gadget_start()
3239 	INIT_LIST_HEAD(&priv_dev->aligned_buf_list);  in cdns3_gadget_start()
3240 	priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",  in cdns3_gadget_start()
3241 						 priv_dev->sysdev,  in cdns3_gadget_start()
3243 	if (!priv_dev->eps_dma_pool) {  in cdns3_gadget_start()
3244 		dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");  in cdns3_gadget_start()
3249 	ret = cdns3_init_eps(priv_dev);  in cdns3_gadget_start()
3251 		dev_err(priv_dev->dev, "Failed to create endpoints\n");  in cdns3_gadget_start()
3256 	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,  in cdns3_gadget_start()
3257 						 &priv_dev->setup_dma, GFP_DMA);  in cdns3_gadget_start()
3258 	if (!priv_dev->setup_buf) {  in cdns3_gadget_start()
3263 	priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);  in cdns3_gadget_start()
3265 	dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",  in cdns3_gadget_start()
3266 		readl(&priv_dev->regs->usb_cap6));  in cdns3_gadget_start()
3267 	dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",  in cdns3_gadget_start()
3268 		readl(&priv_dev->regs->usb_cap1));  in cdns3_gadget_start()
3269 	dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",  in cdns3_gadget_start()
3270 		readl(&priv_dev->regs->usb_cap2));  in cdns3_gadget_start()
3272 	priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);  in cdns3_gadget_start()
3273 	if (priv_dev->dev_ver >= DEV_VER_V2)  in cdns3_gadget_start()
3274 		priv_dev->gadget.sg_supported = 1;  in cdns3_gadget_start()
3276 	priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);  in cdns3_gadget_start()
3277 	if (!priv_dev->zlp_buf) {  in cdns3_gadget_start()
3283 	ret = usb_add_gadget(&priv_dev->gadget);  in cdns3_gadget_start()
3285 		dev_err(priv_dev->dev, "Failed to add gadget\n");  in cdns3_gadget_start()
3291 	kfree(priv_dev->zlp_buf);  in cdns3_gadget_start()
3293 	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,  in cdns3_gadget_start()
3294 			  priv_dev->setup_dma);  in cdns3_gadget_start()
3296 	cdns3_free_all_eps(priv_dev);  in cdns3_gadget_start()
3298 	dma_pool_destroy(priv_dev->eps_dma_pool);  in cdns3_gadget_start()
3300 	usb_put_gadget(&priv_dev->gadget);  in cdns3_gadget_start()
3347 	struct cdns3_device *priv_dev = cdns->gadget_dev;  in cdns3_gadget_suspend()  local
3350 	cdns3_disconnect_gadget(priv_dev);  in cdns3_gadget_suspend()
3353 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_suspend()
3354 	usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);  in cdns3_gadget_suspend()
3355 	cdns3_hw_reset_eps_config(priv_dev);  in cdns3_gadget_suspend()
3358 	writel(0, &priv_dev->regs->usb_ien);  in cdns3_gadget_suspend()
3365 	struct cdns3_device *priv_dev = cdns->gadget_dev;  in cdns3_gadget_resume()  local
3367 	if (!priv_dev->gadget_driver)  in cdns3_gadget_resume()
3370 	cdns3_gadget_config(priv_dev);  in cdns3_gadget_resume()
3372 		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);  in cdns3_gadget_resume()