49 	void (*set_dma_mode)(struct cppi41_dma_channel *cppi41_channel,
54 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)  in save_rx_toggle()  argument
59 	if (cppi41_channel->is_tx)  in save_rx_toggle()
61 	if (!is_host_active(cppi41_channel->controller->controller.musb))  in save_rx_toggle()
64 	csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);  in save_rx_toggle()
67 	cppi41_channel->usb_toggle = toggle;  in save_rx_toggle()
70 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)  in update_rx_toggle()  argument
72 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in update_rx_toggle()
77 	if (cppi41_channel->is_tx)  in update_rx_toggle()
91 	if (!toggle && toggle == cppi41_channel->usb_toggle) {  in update_rx_toggle()
93 		musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);  in update_rx_toggle()
97 	cppi41_channel->usb_toggle = toggle;  in update_rx_toggle()
117 static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)  in cppi41_trans_done()  argument
119 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_trans_done()
124 	if (!cppi41_channel->prog_len ||  in cppi41_trans_done()
125 	    (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) {  in cppi41_trans_done()
128 		cppi41_channel->channel.actual_len =  in cppi41_trans_done()
129 			cppi41_channel->transferred;  in cppi41_trans_done()
130 		cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;  in cppi41_trans_done()
131 		cppi41_channel->channel.rx_packet_done = true;  in cppi41_trans_done()
137 		if (cppi41_channel->tx_zlp && (cppi41_channel->transferred %  in cppi41_trans_done()
138 					cppi41_channel->packet_sz) == 0) {  in cppi41_trans_done()
144 		trace_musb_cppi41_done(cppi41_channel);  in cppi41_trans_done()
145 		musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);  in cppi41_trans_done()
148 		struct dma_chan *dc = cppi41_channel->dc;  in cppi41_trans_done()
153 		cppi41_channel->buf_addr += cppi41_channel->packet_sz;  in cppi41_trans_done()
155 		remain_bytes = cppi41_channel->total_len;  in cppi41_trans_done()
156 		remain_bytes -= cppi41_channel->transferred;  in cppi41_trans_done()
157 		remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);  in cppi41_trans_done()
158 		cppi41_channel->prog_len = remain_bytes;  in cppi41_trans_done()
160 		direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV  in cppi41_trans_done()
163 				cppi41_channel->buf_addr,  in cppi41_trans_done()
171 		dma_desc->callback_param = &cppi41_channel->channel;  in cppi41_trans_done()
172 		cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);  in cppi41_trans_done()
173 		trace_musb_cppi41_cont(cppi41_channel);  in cppi41_trans_done()
176 		if (!cppi41_channel->is_tx) {  in cppi41_trans_done()
188 	struct cppi41_dma_channel *cppi41_channel, *n;  in cppi41_recheck_tx_req()  local
198 	list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,  in cppi41_recheck_tx_req()
201 		struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_recheck_tx_req()
205 			list_del_init(&cppi41_channel->tx_check);  in cppi41_recheck_tx_req()
206 			cppi41_trans_done(cppi41_channel);  in cppi41_recheck_tx_req()
224 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_callback()  local
225 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_dma_callback()
234 	controller = cppi41_channel->controller;  in cppi41_dma_callback()
243 	dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,  in cppi41_dma_callback()
245 	transferred = cppi41_channel->prog_len - txstate.residue;  in cppi41_dma_callback()
246 	cppi41_channel->transferred += transferred;  in cppi41_dma_callback()
248 	trace_musb_cppi41_gb(cppi41_channel);  in cppi41_dma_callback()
249 	update_rx_toggle(cppi41_channel);  in cppi41_dma_callback()
251 	if (cppi41_channel->transferred == cppi41_channel->total_len ||  in cppi41_dma_callback()
252 			transferred < cppi41_channel->packet_sz)  in cppi41_dma_callback()
253 		cppi41_channel->prog_len = 0;  in cppi41_dma_callback()
255 	if (cppi41_channel->is_tx) {  in cppi41_dma_callback()
277 	if (!cppi41_channel->is_tx || empty) {  in cppi41_dma_callback()
278 		cppi41_trans_done(cppi41_channel);  in cppi41_dma_callback()
305 				cppi41_trans_done(cppi41_channel);  in cppi41_dma_callback()
314 	list_add_tail(&cppi41_channel->tx_check,  in cppi41_dma_callback()
317 		unsigned long usecs = cppi41_channel->total_len / 10;  in cppi41_dma_callback()
339 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,  in cppi41_set_dma_mode()  argument
342 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_set_dma_mode()
348 	if (cppi41_channel->is_tx)  in cppi41_set_dma_mode()
352 	port = cppi41_channel->port_num;  in cppi41_set_dma_mode()
357 	if (cppi41_channel->is_tx) {  in cppi41_set_dma_mode()
366 static void da8xx_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,  in da8xx_set_dma_mode()  argument
369 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in da8xx_set_dma_mode()
377 	port = cppi41_channel->port_num;  in da8xx_set_dma_mode()
380 	if (!cppi41_channel->is_tx)  in da8xx_set_dma_mode()
392 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,  in cppi41_set_autoreq_mode()  argument
395 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_set_autoreq_mode()
401 	port = cppi41_channel->port_num;  in cppi41_set_autoreq_mode()
415 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_configure_channel()  local
416 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_configure_channel()
417 	struct dma_chan *dc = cppi41_channel->dc;  in cppi41_configure_channel()
420 	struct musb *musb = cppi41_channel->controller->controller.musb;  in cppi41_configure_channel()
423 	cppi41_channel->buf_addr = dma_addr;  in cppi41_configure_channel()
424 	cppi41_channel->total_len = len;  in cppi41_configure_channel()
425 	cppi41_channel->transferred = 0;  in cppi41_configure_channel()
426 	cppi41_channel->packet_sz = packet_sz;  in cppi41_configure_channel()
427 	cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0;  in cppi41_configure_channel()
433 	if (cppi41_channel->is_tx)  in cppi41_configure_channel()
440 				RNDIS_REG(cppi41_channel->port_num), len);  in cppi41_configure_channel()
442 			controller->set_dma_mode(cppi41_channel,  in cppi41_configure_channel()
446 			cppi41_set_autoreq_mode(cppi41_channel,  in cppi41_configure_channel()
450 					RNDIS_REG(cppi41_channel->port_num), 0);  in cppi41_configure_channel()
451 			controller->set_dma_mode(cppi41_channel,  in cppi41_configure_channel()
453 			cppi41_set_autoreq_mode(cppi41_channel,  in cppi41_configure_channel()
458 		controller->set_dma_mode(cppi41_channel,  in cppi41_configure_channel()
460 		cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);  in cppi41_configure_channel()
463 	cppi41_channel->prog_len = len;  in cppi41_configure_channel()
464 	direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;  in cppi41_configure_channel()
472 	cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);  in cppi41_configure_channel()
473 	cppi41_channel->channel.rx_packet_done = false;  in cppi41_configure_channel()
475 	trace_musb_cppi41_config(cppi41_channel);  in cppi41_configure_channel()
477 	save_rx_toggle(cppi41_channel);  in cppi41_configure_channel()
487 	struct cppi41_dma_channel *cppi41_channel = NULL;  in cppi41_dma_channel_allocate()  local
494 		cppi41_channel = &controller->tx_channel[ch_num];  in cppi41_dma_channel_allocate()
496 		cppi41_channel = &controller->rx_channel[ch_num];  in cppi41_dma_channel_allocate()
498 	if (!cppi41_channel->dc)  in cppi41_dma_channel_allocate()
501 	if (cppi41_channel->is_allocated)  in cppi41_dma_channel_allocate()
504 	cppi41_channel->hw_ep = hw_ep;  in cppi41_dma_channel_allocate()
505 	cppi41_channel->is_allocated = 1;  in cppi41_dma_channel_allocate()
507 	trace_musb_cppi41_alloc(cppi41_channel);  in cppi41_dma_channel_allocate()
508 	return &cppi41_channel->channel;  in cppi41_dma_channel_allocate()
513 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_release()  local
515 	trace_musb_cppi41_free(cppi41_channel);  in cppi41_dma_channel_release()
516 	if (cppi41_channel->is_allocated) {  in cppi41_dma_channel_release()
517 		cppi41_channel->is_allocated = 0;  in cppi41_dma_channel_release()
528 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_program()  local
534 	if (is_host_active(cppi41_channel->controller->controller.musb)) {  in cppi41_dma_channel_program()
535 		if (cppi41_channel->is_tx)  in cppi41_dma_channel_program()
536 			hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;  in cppi41_dma_channel_program()
538 			hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;  in cppi41_dma_channel_program()
557 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_is_compatible()  local
558 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_is_compatible()
565 	if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)  in cppi41_is_compatible()
567 	if (cppi41_channel->is_tx)  in cppi41_is_compatible()
575 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_abort()  local
576 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_dma_channel_abort()
578 	void __iomem *epio = cppi41_channel->hw_ep->regs;  in cppi41_dma_channel_abort()
584 	is_tx = cppi41_channel->is_tx;  in cppi41_dma_channel_abort()
585 	trace_musb_cppi41_abort(cppi41_channel);  in cppi41_dma_channel_abort()
587 	if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)  in cppi41_dma_channel_abort()
590 	list_del_init(&cppi41_channel->tx_check);  in cppi41_dma_channel_abort()
596 		cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);  in cppi41_dma_channel_abort()
620 	tdbit = 1 << cppi41_channel->port_num;  in cppi41_dma_channel_abort()
628 		ret = dmaengine_terminate_all(cppi41_channel->dc);  in cppi41_dma_channel_abort()
641 	cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;  in cppi41_dma_channel_abort()
670 	struct cppi41_dma_channel *cppi41_channel;  in cppi41_dma_controller_start()  local
705 			cppi41_channel = &controller->tx_channel[port - 1];  in cppi41_dma_controller_start()
707 			cppi41_channel = &controller->rx_channel[port - 1];  in cppi41_dma_controller_start()
709 		cppi41_channel->controller = controller;  in cppi41_dma_controller_start()
710 		cppi41_channel->port_num = port;  in cppi41_dma_controller_start()
711 		cppi41_channel->is_tx = is_tx;  in cppi41_dma_controller_start()
712 		INIT_LIST_HEAD(&cppi41_channel->tx_check);  in cppi41_dma_controller_start()
714 		musb_dma = &cppi41_channel->channel;  in cppi41_dma_controller_start()
715 		musb_dma->private_data = cppi41_channel;  in cppi41_dma_controller_start()
728 		cppi41_channel->dc = dc;  in cppi41_dma_controller_start()