37 qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,  in qtd_fill()  argument
45 	qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
46 	qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));  in qtd_fill()
57 			qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
58 			qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,  in qtd_fill()
71 	qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);  in qtd_fill()
80 qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)  in qh_update()  argument
87 	hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_update()
88 	hw->hw_alt_next = EHCI_LIST_END(ehci);  in qh_update()
95 	if (!(hw->hw_info1 & cpu_to_hc32(ehci, QH_TOGGLE_CTL))) {  in qh_update()
99 		epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;  in qh_update()
101 			hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);  in qh_update()
106 	hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);  in qh_update()
114 qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_refresh()  argument
127 	if (qh->hw->hw_token & ACTIVE_BIT(ehci)) {  in qh_refresh()
130 			ehci_warn(ehci, "qh %p should be inactive!\n", qh);  in qh_refresh()
132 		qh_update(ehci, qh, qtd);  in qh_refresh()
139 static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
144 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_clear_tt_buffer_complete()  local
148 	spin_lock_irqsave(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
151 			&& ehci->rh_state == EHCI_RH_RUNNING)  in ehci_clear_tt_buffer_complete()
152 		qh_link_async(ehci, qh);  in ehci_clear_tt_buffer_complete()
153 	spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
156 static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,  in ehci_clear_tt_buffer()  argument
174 		if (!ehci_is_TDI(ehci)  in ehci_clear_tt_buffer()
176 				   ehci_to_hcd(ehci)->self.root_hub) {  in ehci_clear_tt_buffer()
189 	struct ehci_hcd *ehci,  in qtd_copy_status()  argument
238 			ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",  in qtd_copy_status()
252 ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)  in ehci_urb_done()  argument
256 		ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;  in ehci_urb_done()
260 		INCR(ehci->stats.unlink);  in ehci_urb_done()
265 		INCR(ehci->stats.complete);  in ehci_urb_done()
269 	ehci_dbg (ehci,  in ehci_urb_done()
278 	usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in ehci_urb_done()
279 	usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);  in ehci_urb_done()
282 static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
290 qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_completions()  argument
334 				ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
337 			ehci_qtd_free (ehci, last);  in qh_completions()
347 		token = hc32_to_cpu(ehci, qtd->hw_token);  in qh_completions()
355 				ehci_dbg(ehci,  in qh_completions()
376 					ehci_dbg(ehci,  in qh_completions()
388 					qtd->hw_token = cpu_to_hc32(ehci,  in qh_completions()
391 					hw->hw_token = cpu_to_hc32(ehci,  in qh_completions()
409 						& EHCI_LIST_END(ehci))) {  in qh_completions()
416 				&& ehci->rh_state >= EHCI_RH_RUNNING)) {  in qh_completions()
424 			if (ehci->rh_state < EHCI_RH_RUNNING) {  in qh_completions()
445 					(hw->hw_token & ACTIVE_BIT(ehci))) {  in qh_completions()
446 				token = hc32_to_cpu(ehci, hw->hw_token);  in qh_completions()
447 				hw->hw_token &= ~ACTIVE_BIT(ehci);  in qh_completions()
454 				ehci_clear_tt_buffer(ehci, qh, urb, token);  in qh_completions()
466 			last_status = qtd_copy_status(ehci, urb,  in qh_completions()
470 						& EHCI_LIST_END(ehci)))  in qh_completions()
488 					ehci_clear_tt_buffer(ehci, qh, urb,  in qh_completions()
512 		ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
513 		ehci_qtd_free (ehci, last);  in qh_completions()
543 	if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci))  in qh_completions()
557 	struct ehci_hcd		*ehci,  in qtd_list_free()  argument
568 		ehci_qtd_free (ehci, qtd);  in qtd_list_free()
577 	struct ehci_hcd		*ehci,  in qh_urb_transaction()  argument
593 	qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
607 		qtd_fill(ehci, qtd, urb->setup_dma,  in qh_urb_transaction()
614 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
618 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
658 		this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,  in qh_urb_transaction()
670 			qtd->hw_alt_next = ehci->async->hw->hw_alt_next;  in qh_urb_transaction()
685 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
689 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
700 		qtd->hw_alt_next = EHCI_LIST_END(ehci);  in qh_urb_transaction()
721 			qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
725 			qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
729 			qtd_fill(ehci, qtd, 0, 0, token, 0);  in qh_urb_transaction()
735 		qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);  in qh_urb_transaction()
739 	qtd_list_free (ehci, urb, head);  in qh_urb_transaction()
762 	struct ehci_hcd		*ehci,  in qh_make()  argument
766 	struct ehci_qh		*qh = ehci_qh_alloc (ehci, flags);  in qh_make()
794 		ehci_dbg(ehci, "bogus qh maxpacket %d\n", maxp);  in qh_make()
823 			} else if (urb->interval > ehci->periodic_size << 3) {  in qh_make()
824 				urb->interval = ehci->periodic_size << 3;  in qh_make()
855 			if (urb->interval > ehci->periodic_size)  in qh_make()
856 				urb->interval = ehci->periodic_size;  in qh_make()
895 		if (ehci_has_fsl_portno_bug(ehci))  in qh_make()
903 		if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)  in qh_make()
933 		ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev,  in qh_make()
936 		qh_destroy(ehci, qh);  in qh_make()
945 	hw->hw_info1 = cpu_to_hc32(ehci, info1);  in qh_make()
946 	hw->hw_info2 = cpu_to_hc32(ehci, info2);  in qh_make()
954 static void enable_async(struct ehci_hcd *ehci)  in enable_async()  argument
956 	if (ehci->async_count++)  in enable_async()
960 	ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC);  in enable_async()
963 	ehci_poll_ASS(ehci);  in enable_async()
964 	turn_on_io_watchdog(ehci);  in enable_async()
967 static void disable_async(struct ehci_hcd *ehci)  in disable_async()  argument
969 	if (--ehci->async_count)  in disable_async()
973 	WARN_ON(ehci->async->qh_next.qh || !list_empty(&ehci->async_unlink) ||  in disable_async()
974 			!list_empty(&ehci->async_idle));  in disable_async()
977 	ehci_poll_ASS(ehci);  in disable_async()
982 static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_link_async()  argument
984 	__hc32		dma = QH_NEXT(ehci, qh->qh_dma);  in qh_link_async()
994 	qh_refresh(ehci, qh);  in qh_link_async()
997 	head = ehci->async;  in qh_link_async()
1010 	enable_async(ehci);  in qh_link_async()
1022 	struct ehci_hcd		*ehci,  in qh_append_tds()  argument
1030 	__hc32			qh_addr_mask = cpu_to_hc32(ehci, 0x7f);  in qh_append_tds()
1035 		qh = qh_make (ehci, urb, GFP_ATOMIC);  in qh_append_tds()
1069 			qtd->hw_token = HALT_BIT(ehci);  in qh_append_tds()
1081 			ehci_qtd_init(ehci, qtd, qtd->qtd_dma);  in qh_append_tds()
1088 			qtd->hw_next = QTD_NEXT(ehci, dma);  in qh_append_tds()
1104 	struct ehci_hcd		*ehci,  in submit_async()  argument
1120 		ehci_dbg(ehci,  in submit_async()
1129 	spin_lock_irqsave (&ehci->lock, flags);  in submit_async()
1130 	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {  in submit_async()
1134 	rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1138 	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);  in submit_async()
1140 		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1149 		qh_link_async(ehci, qh);  in submit_async()
1151 	spin_unlock_irqrestore (&ehci->lock, flags);  in submit_async()
1153 		qtd_list_free (ehci, urb, qtd_list);  in submit_async()
1174 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_submit_single_step_set_feature()  local
1187 	qtd = ehci_qtd_alloc(ehci, GFP_KERNEL);  in ehci_submit_single_step_set_feature()
1204 		qtd_fill(ehci, qtd, urb->setup_dma,  in ehci_submit_single_step_set_feature()
1208 		submit_async(ehci, urb, &qtd_list, GFP_ATOMIC);  in ehci_submit_single_step_set_feature()
1223 	qtd_fill(ehci, qtd, buf, len, token, maxpacket);  in ehci_submit_single_step_set_feature()
1229 	qtd->hw_alt_next = EHCI_LIST_END(ehci);  in ehci_submit_single_step_set_feature()
1236 	qtd = ehci_qtd_alloc(ehci, GFP_ATOMIC);  in ehci_submit_single_step_set_feature()
1240 	qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in ehci_submit_single_step_set_feature()
1244 	qtd_fill(ehci, qtd, 0, 0, token | QTD_IOC, 0);  in ehci_submit_single_step_set_feature()
1246 	submit_async(ehci, urb, &qtd_list, GFP_KERNEL);  in ehci_submit_single_step_set_feature()
1251 	qtd_list_free(ehci, urb, head);  in ehci_submit_single_step_set_feature()
1258 static void single_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)  in single_unlink_async()  argument
1264 	list_add_tail(&qh->unlink_node, &ehci->async_unlink);  in single_unlink_async()
1267 	prev = ehci->async;  in single_unlink_async()
1273 	if (ehci->qh_scan_next == qh)  in single_unlink_async()
1274 		ehci->qh_scan_next = qh->qh_next.qh;  in single_unlink_async()
1277 static void start_iaa_cycle(struct ehci_hcd *ehci)  in start_iaa_cycle()  argument
1280 	if (unlikely(ehci->rh_state < EHCI_RH_RUNNING)) {  in start_iaa_cycle()
1281 		end_unlink_async(ehci);  in start_iaa_cycle()
1284 	} else if (ehci->rh_state == EHCI_RH_RUNNING &&  in start_iaa_cycle()
1285 			!ehci->iaa_in_progress) {  in start_iaa_cycle()
1290 		ehci_writel(ehci, ehci->command | CMD_IAAD,  in start_iaa_cycle()
1291 				&ehci->regs->command);  in start_iaa_cycle()
1292 		ehci_readl(ehci, &ehci->regs->command);  in start_iaa_cycle()
1293 		ehci->iaa_in_progress = true;  in start_iaa_cycle()
1294 		ehci_enable_event(ehci, EHCI_HRTIMER_IAA_WATCHDOG, true);  in start_iaa_cycle()
1298 static void end_iaa_cycle(struct ehci_hcd *ehci)  in end_iaa_cycle()  argument
1300 	if (ehci->has_synopsys_hc_bug)  in end_iaa_cycle()
1301 		ehci_writel(ehci, (u32) ehci->async->qh_dma,  in end_iaa_cycle()
1302 			    &ehci->regs->async_next);  in end_iaa_cycle()
1305 	ehci->iaa_in_progress = false;  in end_iaa_cycle()
1307 	end_unlink_async(ehci);  in end_iaa_cycle()
1312 static void end_unlink_async(struct ehci_hcd *ehci)  in end_unlink_async()  argument
1317 	if (list_empty(&ehci->async_unlink))  in end_unlink_async()
1319 	qh = list_first_entry(&ehci->async_unlink, struct ehci_qh,  in end_unlink_async()
1326 	early_exit = ehci->async_unlinking;  in end_unlink_async()
1329 	if (ehci->rh_state < EHCI_RH_RUNNING)  in end_unlink_async()
1330 		list_splice_tail_init(&ehci->async_unlink, &ehci->async_idle);  in end_unlink_async()
1342 		list_move_tail(&qh->unlink_node, &ehci->async_idle);  in end_unlink_async()
1369 	else if	(qh->hw->hw_token & cpu_to_hc32(ehci, QTD_STS_HALT))  in end_unlink_async()
1378 		if (qh_current != ehci->old_current ||  in end_unlink_async()
1379 				qh_token != ehci->old_token) {  in end_unlink_async()
1380 			ehci->old_current = qh_current;  in end_unlink_async()
1381 			ehci->old_token = qh_token;  in end_unlink_async()
1382 			ehci_enable_event(ehci,  in end_unlink_async()
1390 	ehci->old_current = ~0;		/* Prepare for next QH */  in end_unlink_async()
1393 	if (!list_empty(&ehci->async_unlink))  in end_unlink_async()
1394 		start_iaa_cycle(ehci);  in end_unlink_async()
1404 	ehci->async_unlinking = true;  in end_unlink_async()
1405 	while (!list_empty(&ehci->async_idle)) {  in end_unlink_async()
1406 		qh = list_first_entry(&ehci->async_idle, struct ehci_qh,  in end_unlink_async()
1414 			qh_completions(ehci, qh);  in end_unlink_async()
1416 				ehci->rh_state == EHCI_RH_RUNNING)  in end_unlink_async()
1417 			qh_link_async(ehci, qh);  in end_unlink_async()
1418 		disable_async(ehci);  in end_unlink_async()
1420 	ehci->async_unlinking = false;  in end_unlink_async()
1423 static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
1425 static void unlink_empty_async(struct ehci_hcd *ehci)  in unlink_empty_async()  argument
1432 	for (qh = ehci->async->qh_next.qh; qh; qh = qh->qh_next.qh) {  in unlink_empty_async()
1436 			if (qh->unlink_cycle != ehci->async_unlink_cycle)  in unlink_empty_async()
1442 	if (list_empty(&ehci->async_unlink) && qh_to_unlink) {  in unlink_empty_async()
1444 		start_unlink_async(ehci, qh_to_unlink);  in unlink_empty_async()
1450 		ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);  in unlink_empty_async()
1451 		++ehci->async_unlink_cycle;  in unlink_empty_async()
1458 static void unlink_empty_async_suspended(struct ehci_hcd *ehci)  in unlink_empty_async_suspended()  argument
1462 	while (ehci->async->qh_next.qh) {  in unlink_empty_async_suspended()
1463 		qh = ehci->async->qh_next.qh;  in unlink_empty_async_suspended()
1465 		single_unlink_async(ehci, qh);  in unlink_empty_async_suspended()
1474 static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)  in start_unlink_async()  argument
1480 	single_unlink_async(ehci, qh);  in start_unlink_async()
1481 	start_iaa_cycle(ehci);  in start_unlink_async()
1486 static void scan_async (struct ehci_hcd *ehci)  in scan_async()  argument
1491 	ehci->qh_scan_next = ehci->async->qh_next.qh;  in scan_async()
1492 	while (ehci->qh_scan_next) {  in scan_async()
1493 		qh = ehci->qh_scan_next;  in scan_async()
1494 		ehci->qh_scan_next = qh->qh_next.qh;  in scan_async()
1507 			temp = qh_completions(ehci, qh);  in scan_async()
1509 				start_unlink_async(ehci, qh);  in scan_async()
1512 				qh->unlink_cycle = ehci->async_unlink_cycle;  in scan_async()
1524 	if (check_unlinks_later && ehci->rh_state == EHCI_RH_RUNNING &&  in scan_async()
1525 			!(ehci->enabled_hrtimer_events &  in scan_async()
1527 		ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);  in scan_async()
1528 		++ehci->async_unlink_cycle;  in scan_async()