46 qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,  in qtd_fill()  argument
53 	qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
54 	qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));  in qtd_fill()
65 			qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
66 			qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,  in qtd_fill()
79 	qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);  in qtd_fill()
88 qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)  in qh_update()  argument
95 	hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_update()
96 	hw->hw_alt_next = EHCI_LIST_END(ehci);  in qh_update()
103 	if (!(hw->hw_info1 & cpu_to_hc32(ehci, 1 << 14))) {  in qh_update()
107 		epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;  in qh_update()
109 			hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);  in qh_update()
114 	hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);  in qh_update()
122 qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_refresh()  argument
138 		if (cpu_to_hc32(ehci, qtd->qtd_dma) == qh->hw->hw_current) {  in qh_refresh()
145 		qh_update (ehci, qh, qtd);  in qh_refresh()
150 static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
155 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_clear_tt_buffer_complete()  local
159 	spin_lock_irqsave(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
162 			&& ehci->rh_state == EHCI_RH_RUNNING)  in ehci_clear_tt_buffer_complete()
163 		qh_link_async(ehci, qh);  in ehci_clear_tt_buffer_complete()
164 	spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
167 static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,  in ehci_clear_tt_buffer()  argument
185 		if (!ehci_is_TDI(ehci)  in ehci_clear_tt_buffer()
187 				   ehci_to_hcd(ehci)->self.root_hub) {  in ehci_clear_tt_buffer()
200 	struct ehci_hcd *ehci,  in qtd_copy_status()  argument
242 			ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",  in qtd_copy_status()
251 		ehci_vdbg (ehci,  in qtd_copy_status()
263 ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)  in ehci_urb_done()  argument
264 __releases(ehci->lock)  in ehci_urb_done()
265 __acquires(ehci->lock)  in ehci_urb_done()
269 		ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;  in ehci_urb_done()
276 		COUNT(ehci->stats.unlink);  in ehci_urb_done()
281 		COUNT(ehci->stats.complete);  in ehci_urb_done()
285 	ehci_dbg (ehci,  in ehci_urb_done()
295 	usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in ehci_urb_done()
296 	spin_unlock (&ehci->lock);  in ehci_urb_done()
297 	usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);  in ehci_urb_done()
298 	spin_lock (&ehci->lock);  in ehci_urb_done()
301 static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
302 static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
304 static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
312 qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_completions()  argument
360 				ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
364 			ehci_qtd_free (ehci, last);  in qh_completions()
374 		token = hc32_to_cpu(ehci, qtd->hw_token);  in qh_completions()
382 				ehci_dbg(ehci,  in qh_completions()
403 					ehci_dbg(ehci,  in qh_completions()
415 					qtd->hw_token = cpu_to_hc32(ehci,  in qh_completions()
418 					hw->hw_token = cpu_to_hc32(ehci,  in qh_completions()
435 						& EHCI_LIST_END(ehci))) {  in qh_completions()
441 				&& ehci->rh_state == EHCI_RH_RUNNING)) {  in qh_completions()
449 			if (ehci->rh_state != EHCI_RH_RUNNING)  in qh_completions()
460 					&& cpu_to_hc32(ehci, qtd->qtd_dma)  in qh_completions()
462 				token = hc32_to_cpu(ehci, hw->hw_token);  in qh_completions()
468 				ehci_clear_tt_buffer(ehci, qh, urb, token);  in qh_completions()
480 			last_status = qtd_copy_status(ehci, urb,  in qh_completions()
484 						& EHCI_LIST_END(ehci)))  in qh_completions()
502 					ehci_clear_tt_buffer(ehci, qh, urb,  in qh_completions()
526 		ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
528 		ehci_qtd_free (ehci, last);  in qh_completions()
553 	if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) {  in qh_completions()
556 			qh_refresh(ehci, qh);  in qh_completions()
593 	struct ehci_hcd		*ehci,  in qtd_list_free()  argument
604 		ehci_qtd_free (ehci, qtd);  in qtd_list_free()
613 	struct ehci_hcd		*ehci,  in qh_urb_transaction()  argument
629 	qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
643 		qtd_fill(ehci, qtd, urb->setup_dma,  in qh_urb_transaction()
650 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
654 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
694 		this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,  in qh_urb_transaction()
706 			qtd->hw_alt_next = ehci->async->hw->hw_alt_next;  in qh_urb_transaction()
721 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
725 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
736 		qtd->hw_alt_next = EHCI_LIST_END(ehci);  in qh_urb_transaction()
757 			qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
761 			qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
765 			qtd_fill(ehci, qtd, 0, 0, token, 0);  in qh_urb_transaction()
771 		qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);  in qh_urb_transaction()
775 	qtd_list_free (ehci, urb, head);  in qh_urb_transaction()
798 	struct ehci_hcd		*ehci,  in qh_make()  argument
802 	struct ehci_qh		*qh = ehci_qh_alloc (ehci, flags);  in qh_make()
826 		ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));  in qh_make()
843 		qh->stamp = ehci->periodic_stamp;  in qh_make()
856 			} else if (qh->period > ehci->periodic_size) {  in qh_make()
857 				qh->period = ehci->periodic_size;  in qh_make()
881 			if (qh->period > ehci->periodic_size) {  in qh_make()
882 				qh->period = ehci->periodic_size;  in qh_make()
912 		if (ehci_has_fsl_portno_bug(ehci))  in qh_make()
920 		if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)  in qh_make()
961 	hw->hw_info1 = cpu_to_hc32(ehci, info1);  in qh_make()
962 	hw->hw_info2 = cpu_to_hc32(ehci, info2);  in qh_make()
965 	qh_refresh (ehci, qh);  in qh_make()
973 static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_link_async()  argument
975 	__hc32		dma = QH_NEXT(ehci, qh->qh_dma);  in qh_link_async()
985 	head = ehci->async;  in qh_link_async()
986 	timer_action_done (ehci, TIMER_ASYNC_OFF);  in qh_link_async()
988 		u32	cmd = ehci_readl(ehci, &ehci->regs->command);  in qh_link_async()
992 			(void)handshake(ehci, &ehci->regs->status,  in qh_link_async()
995 			ehci_writel(ehci, cmd, &ehci->regs->command);  in qh_link_async()
1001 	qh_refresh(ehci, qh);  in qh_link_async()
1026 	struct ehci_hcd		*ehci,  in qh_append_tds()  argument
1034 	__hc32			qh_addr_mask = cpu_to_hc32(ehci, 0x7f);  in qh_append_tds()
1039 		qh = qh_make (ehci, urb, GFP_ATOMIC);  in qh_append_tds()
1073 			qtd->hw_token = HALT_BIT(ehci);  in qh_append_tds()
1085 			ehci_qtd_init(ehci, qtd, qtd->qtd_dma);  in qh_append_tds()
1092 			qtd->hw_next = QTD_NEXT(ehci, dma);  in qh_append_tds()
1108 	struct ehci_hcd		*ehci,  in submit_async()  argument
1124 		ehci_dbg(ehci,  in submit_async()
1133 	spin_lock_irqsave (&ehci->lock, flags);  in submit_async()
1134 	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {  in submit_async()
1138 	rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1142 	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);  in submit_async()
1144 		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1153 		qh_link_async(ehci, qh);  in submit_async()
1155 	spin_unlock_irqrestore (&ehci->lock, flags);  in submit_async()
1157 		qtd_list_free (ehci, urb, qtd_list);  in submit_async()
1165 static void end_unlink_async (struct ehci_hcd *ehci)  in end_unlink_async()  argument
1167 	struct ehci_qh		*qh = ehci->reclaim;  in end_unlink_async()
1170 	iaa_watchdog_done(ehci);  in end_unlink_async()
1179 	ehci->reclaim = next;  in end_unlink_async()
1182 	qh_completions (ehci, qh);  in end_unlink_async()
1184 	if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {  in end_unlink_async()
1185 		qh_link_async (ehci, qh);  in end_unlink_async()
1190 		if (ehci->rh_state == EHCI_RH_RUNNING  in end_unlink_async()
1191 				&& ehci->async->qh_next.qh == NULL)  in end_unlink_async()
1192 			timer_action (ehci, TIMER_ASYNC_OFF);  in end_unlink_async()
1197 		ehci->reclaim = NULL;  in end_unlink_async()
1198 		start_unlink_async (ehci, next);  in end_unlink_async()
1201 	if (ehci->has_synopsys_hc_bug)  in end_unlink_async()
1202 		ehci_writel(ehci, (u32) ehci->async->qh_dma,  in end_unlink_async()
1203 			    &ehci->regs->async_next);  in end_unlink_async()
1209 static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh)  in start_unlink_async()  argument
1211 	int		cmd = ehci_readl(ehci, &ehci->regs->command);  in start_unlink_async()
1215 	assert_spin_locked(&ehci->lock);  in start_unlink_async()
1216 	if (ehci->reclaim  in start_unlink_async()
1224 	if (unlikely (qh == ehci->async)) {  in start_unlink_async()
1226 		if (ehci->rh_state != EHCI_RH_HALTED  in start_unlink_async()
1227 				&& !ehci->reclaim) {  in start_unlink_async()
1229 			ehci_writel(ehci, cmd & ~CMD_ASE,  in start_unlink_async()
1230 				    &ehci->regs->command);  in start_unlink_async()
1233 			timer_action_done (ehci, TIMER_ASYNC_OFF);  in start_unlink_async()
1239 	ehci->reclaim = qh = qh_get (qh);  in start_unlink_async()
1241 	prev = ehci->async;  in start_unlink_async()
1247 	if (ehci->qh_scan_next == qh)  in start_unlink_async()
1248 		ehci->qh_scan_next = qh->qh_next.qh;  in start_unlink_async()
1252 	if (unlikely(ehci->rh_state != EHCI_RH_RUNNING)) {  in start_unlink_async()
1256 		end_unlink_async (ehci);  in start_unlink_async()
1261 	ehci_writel(ehci, cmd, &ehci->regs->command);  in start_unlink_async()
1262 	(void)ehci_readl(ehci, &ehci->regs->command);  in start_unlink_async()
1263 	iaa_watchdog_start(ehci);  in start_unlink_async()
1268 static void scan_async (struct ehci_hcd *ehci)  in scan_async()  argument
1274 	timer_action_done (ehci, TIMER_ASYNC_SHRINK);  in scan_async()
1275 	stopped = (ehci->rh_state != EHCI_RH_RUNNING);  in scan_async()
1277 	ehci->qh_scan_next = ehci->async->qh_next.qh;  in scan_async()
1278 	while (ehci->qh_scan_next) {  in scan_async()
1279 		qh = ehci->qh_scan_next;  in scan_async()
1280 		ehci->qh_scan_next = qh->qh_next.qh;  in scan_async()
1294 			temp = qh_completions(ehci, qh);  in scan_async()
1296 				unlink_async(ehci, qh);  in scan_async()
1311 			if (!ehci->reclaim && (stopped ||  in scan_async()
1313 				start_unlink_async(ehci, qh);  in scan_async()
1319 		timer_action (ehci, TIMER_ASYNC_SHRINK);  in scan_async()