185 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_terminate_all()  local
190 	dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch);  in moxart_terminate_all()
192 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_terminate_all()
194 	if (ch->desc) {  in moxart_terminate_all()
195 		moxart_dma_desc_free(&ch->desc->vd);  in moxart_terminate_all()
196 		ch->desc = NULL;  in moxart_terminate_all()
199 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
201 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
203 	vchan_get_all_descriptors(&ch->vc, &head);  in moxart_terminate_all()
204 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_terminate_all()
205 	vchan_dma_desc_free_list(&ch->vc, &head);  in moxart_terminate_all()
213 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_slave_config()  local
216 	ch->cfg = *cfg;  in moxart_slave_config()
218 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_slave_config()
223 	switch (ch->cfg.src_addr_width) {  in moxart_slave_config()
226 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
233 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
240 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
249 	if (ch->cfg.direction == DMA_MEM_TO_DEV) {  in moxart_slave_config()
252 		ctrl |= (ch->line_reqno << 16 &  in moxart_slave_config()
257 		ctrl |= (ch->line_reqno << 24 &  in moxart_slave_config()
261 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_slave_config()
271 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_prep_slave_sg()  local
286 		dev_addr = ch->cfg.src_addr;  in moxart_prep_slave_sg()
287 		dev_width = ch->cfg.src_addr_width;  in moxart_prep_slave_sg()
289 		dev_addr = ch->cfg.dst_addr;  in moxart_prep_slave_sg()
290 		dev_width = ch->cfg.dst_addr_width;  in moxart_prep_slave_sg()
324 	ch->error = 0;  in moxart_prep_slave_sg()
326 	return vchan_tx_prep(&ch->vc, &d->vd, tx_flags);  in moxart_prep_slave_sg()
334 	struct moxart_chan *ch;  in moxart_of_xlate()  local
340 	ch = to_moxart_dma_chan(chan);  in moxart_of_xlate()
341 	ch->line_reqno = dma_spec->args[0];  in moxart_of_xlate()
348 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_alloc_chan_resources()  local
351 		__func__, ch->ch_num);  in moxart_alloc_chan_resources()
352 	ch->allocated = 1;  in moxart_alloc_chan_resources()
359 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_free_chan_resources()  local
361 	vchan_free_chan_resources(&ch->vc);  in moxart_free_chan_resources()
364 		__func__, ch->ch_num);  in moxart_free_chan_resources()
365 	ch->allocated = 0;  in moxart_free_chan_resources()
368 static void moxart_dma_set_params(struct moxart_chan *ch, dma_addr_t src_addr,  in moxart_dma_set_params()  argument
371 	writel(src_addr, ch->base + REG_OFF_ADDRESS_SOURCE);  in moxart_dma_set_params()
372 	writel(dst_addr, ch->base + REG_OFF_ADDRESS_DEST);  in moxart_dma_set_params()
375 static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len)  in moxart_set_transfer_params()  argument
377 	struct moxart_desc *d = ch->desc;  in moxart_set_transfer_params()
386 	writel(d->dma_cycles, ch->base + REG_OFF_CYCLES);  in moxart_set_transfer_params()
388 	dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n",  in moxart_set_transfer_params()
392 static void moxart_start_dma(struct moxart_chan *ch)  in moxart_start_dma()  argument
396 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_start_dma()
398 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_start_dma()
401 static void moxart_dma_start_sg(struct moxart_chan *ch, unsigned int idx)  in moxart_dma_start_sg()  argument
403 	struct moxart_desc *d = ch->desc;  in moxart_dma_start_sg()
404 	struct moxart_sg *sg = ch->desc->sg + idx;  in moxart_dma_start_sg()
406 	if (ch->desc->dma_dir == DMA_MEM_TO_DEV)  in moxart_dma_start_sg()
407 		moxart_dma_set_params(ch, sg->addr, d->dev_addr);  in moxart_dma_start_sg()
408 	else if (ch->desc->dma_dir == DMA_DEV_TO_MEM)  in moxart_dma_start_sg()
409 		moxart_dma_set_params(ch, d->dev_addr, sg->addr);  in moxart_dma_start_sg()
411 	moxart_set_transfer_params(ch, sg->len);  in moxart_dma_start_sg()
413 	moxart_start_dma(ch);  in moxart_dma_start_sg()
418 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_dma_start_desc()  local
421 	vd = vchan_next_desc(&ch->vc);  in moxart_dma_start_desc()
424 		ch->desc = NULL;  in moxart_dma_start_desc()
430 	ch->desc = to_moxart_dma_desc(&vd->tx);  in moxart_dma_start_desc()
431 	ch->sgidx = 0;  in moxart_dma_start_desc()
433 	moxart_dma_start_sg(ch, 0);  in moxart_dma_start_desc()
438 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_issue_pending()  local
441 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_issue_pending()
442 	if (vchan_issue_pending(&ch->vc) && !ch->desc)  in moxart_issue_pending()
444 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_issue_pending()
459 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch)  in moxart_dma_desc_size_in_flight()  argument
464 	size = moxart_dma_desc_size(ch->desc, ch->sgidx);  in moxart_dma_desc_size_in_flight()
465 	cycles = readl(ch->base + REG_OFF_CYCLES);  in moxart_dma_desc_size_in_flight()
466 	completed_cycles = (ch->desc->dma_cycles - cycles);  in moxart_dma_desc_size_in_flight()
467 	size -= completed_cycles << es_bytes[ch->desc->es];  in moxart_dma_desc_size_in_flight()
469 	dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size);  in moxart_dma_desc_size_in_flight()
478 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_tx_status()  local
489 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_tx_status()
490 	vd = vchan_find_desc(&ch->vc, cookie);  in moxart_tx_status()
494 	} else if (ch->desc && ch->desc->vd.tx.cookie == cookie) {  in moxart_tx_status()
495 		txstate->residue = moxart_dma_desc_size_in_flight(ch);  in moxart_tx_status()
497 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_tx_status()
499 	if (ch->error)  in moxart_tx_status()
522 	struct moxart_chan *ch = &mc->slave_chans[0];  in moxart_dma_interrupt()  local
526 	dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);  in moxart_dma_interrupt()
528 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_dma_interrupt()
529 		if (!ch->allocated)  in moxart_dma_interrupt()
532 		ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
534 		dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n",  in moxart_dma_interrupt()
535 			__func__, ch, ch->base, ctrl);  in moxart_dma_interrupt()
539 			if (ch->desc) {  in moxart_dma_interrupt()
540 				spin_lock(&ch->vc.lock);  in moxart_dma_interrupt()
541 				if (++ch->sgidx < ch->desc->sglen) {  in moxart_dma_interrupt()
542 					moxart_dma_start_sg(ch, ch->sgidx);  in moxart_dma_interrupt()
544 					vchan_cookie_complete(&ch->desc->vd);  in moxart_dma_interrupt()
545 					moxart_dma_start_desc(&ch->vc.chan);  in moxart_dma_interrupt()
547 				spin_unlock(&ch->vc.lock);  in moxart_dma_interrupt()
553 			ch->error = 1;  in moxart_dma_interrupt()
556 		writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
569 	struct moxart_chan *ch;  in moxart_probe()  local
592 	ch = &mdc->slave_chans[0];  in moxart_probe()
593 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_probe()
594 		ch->ch_num = i;  in moxart_probe()
595 		ch->base = dma_base_addr + i * REG_OFF_CHAN_SIZE;  in moxart_probe()
596 		ch->allocated = 0;  in moxart_probe()
598 		ch->vc.desc_free = moxart_dma_desc_free;  in moxart_probe()
599 		vchan_init(&ch->vc, &mdc->dma_slave);  in moxart_probe()
602 			__func__, i, ch->ch_num, ch->base);  in moxart_probe()