55 #define rh_reg_read(r)		readl(rh->regs + (RH_##r))
56 #define rh_reg_write(r, v)	writel(v, rh->regs + (RH_##r))
168 	struct hci_rh_data *rh;  in hci_dma_cleanup()  local
175 		rh = &rings->headers[i];  in hci_dma_cleanup()
182 		if (rh->xfer)  in hci_dma_cleanup()
184 					  rh->xfer_struct_sz * rh->xfer_entries,  in hci_dma_cleanup()
185 					  rh->xfer, rh->xfer_dma);  in hci_dma_cleanup()
186 		if (rh->resp)  in hci_dma_cleanup()
188 					  rh->resp_struct_sz * rh->xfer_entries,  in hci_dma_cleanup()
189 					  rh->resp, rh->resp_dma);  in hci_dma_cleanup()
190 		kfree(rh->src_xfers);  in hci_dma_cleanup()
191 		if (rh->ibi_status)  in hci_dma_cleanup()
193 					  rh->ibi_status_sz * rh->ibi_status_entries,  in hci_dma_cleanup()
194 					  rh->ibi_status, rh->ibi_status_dma);  in hci_dma_cleanup()
195 		if (rh->ibi_data_dma)  in hci_dma_cleanup()
196 			dma_unmap_single(&hci->master.dev, rh->ibi_data_dma,  in hci_dma_cleanup()
197 					 rh->ibi_chunk_sz * rh->ibi_chunks_total,  in hci_dma_cleanup()
199 		kfree(rh->ibi_data);  in hci_dma_cleanup()
211 	struct hci_rh_data *rh;  in hci_dma_init()  local
239 		rh = &rings->headers[i];  in hci_dma_init()
240 		rh->regs = hci->base_regs + offset;  in hci_dma_init()
241 		spin_lock_init(&rh->lock);  in hci_dma_init()
242 		init_completion(&rh->op_done);  in hci_dma_init()
244 		rh->xfer_entries = XFER_RING_ENTRIES;  in hci_dma_init()
247 		rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval);  in hci_dma_init()
248 		rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval);  in hci_dma_init()
250 		    rh->xfer_struct_sz, rh->resp_struct_sz);  in hci_dma_init()
251 		xfers_sz = rh->xfer_struct_sz * rh->xfer_entries;  in hci_dma_init()
252 		resps_sz = rh->resp_struct_sz * rh->xfer_entries;  in hci_dma_init()
254 		rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz,  in hci_dma_init()
255 					      &rh->xfer_dma, GFP_KERNEL);  in hci_dma_init()
256 		rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz,  in hci_dma_init()
257 					      &rh->resp_dma, GFP_KERNEL);  in hci_dma_init()
258 		rh->src_xfers =  in hci_dma_init()
259 			kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers),  in hci_dma_init()
262 		if (!rh->xfer || !rh->resp || !rh->src_xfers)  in hci_dma_init()
265 		rh_reg_write(CMD_RING_BASE_LO, lo32(rh->xfer_dma));  in hci_dma_init()
266 		rh_reg_write(CMD_RING_BASE_HI, hi32(rh->xfer_dma));  in hci_dma_init()
267 		rh_reg_write(RESP_RING_BASE_LO, lo32(rh->resp_dma));  in hci_dma_init()
268 		rh_reg_write(RESP_RING_BASE_HI, hi32(rh->resp_dma));  in hci_dma_init()
270 		regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries);  in hci_dma_init()
288 		rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval);  in hci_dma_init()
289 		rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES;  in hci_dma_init()
290 		rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE;  in hci_dma_init()
292 		rh->ibi_chunk_sz = dma_get_cache_alignment();  in hci_dma_init()
293 		rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES;  in hci_dma_init()
294 		BUG_ON(rh->ibi_chunk_sz > 256);  in hci_dma_init()
296 		ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries;  in hci_dma_init()
297 		ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total;  in hci_dma_init()
299 		rh->ibi_status =  in hci_dma_init()
301 					   &rh->ibi_status_dma, GFP_KERNEL);  in hci_dma_init()
302 		rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL);  in hci_dma_init()
304 		if (!rh->ibi_status || !rh->ibi_data)  in hci_dma_init()
306 		rh->ibi_data_dma =  in hci_dma_init()
307 			dma_map_single(&hci->master.dev, rh->ibi_data,  in hci_dma_init()
309 		if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) {  in hci_dma_init()
310 			rh->ibi_data_dma = 0;  in hci_dma_init()
316 				    rh->ibi_status_entries) |  in hci_dma_init()
318 				    ilog2(rh->ibi_chunk_sz) - 2) |  in hci_dma_init()
320 				    rh->ibi_chunks_total);  in hci_dma_init()
358 	struct hci_rh_data *rh;  in hci_dma_queue_xfer()  local
364 	rh = &rings->headers[ring];  in hci_dma_queue_xfer()
370 		u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr;  in hci_dma_queue_xfer()
409 		rh->src_xfers[enqueue_ptr] = xfer;  in hci_dma_queue_xfer()
414 		enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries;  in hci_dma_queue_xfer()
429 	spin_lock_irq(&rh->lock);  in hci_dma_queue_xfer()
434 	spin_unlock_irq(&rh->lock);  in hci_dma_queue_xfer()
443 	struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number];  in hci_dma_dequeue_xfer()  local
449 	if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) {  in hci_dma_dequeue_xfer()
469 			u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx;  in hci_dma_dequeue_xfer()
480 			rh->src_xfers[idx] = NULL;  in hci_dma_dequeue_xfer()
495 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh)  in hci_dma_xfer_done()  argument
498 	unsigned int tid, done_ptr = rh->done_ptr;  in hci_dma_xfer_done()
506 		ring_resp = rh->resp + rh->resp_struct_sz * done_ptr;  in hci_dma_xfer_done()
511 		xfer = rh->src_xfers[done_ptr];  in hci_dma_xfer_done()
528 		done_ptr = (done_ptr + 1) % rh->xfer_entries;  in hci_dma_xfer_done()
529 		rh->done_ptr = done_ptr;  in hci_dma_xfer_done()
533 	spin_lock(&rh->lock);  in hci_dma_xfer_done()
538 	spin_unlock(&rh->lock);  in hci_dma_xfer_done()
582 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)  in hci_dma_process_ibi()  argument
609 	     ptr = (ptr + 1) % rh->ibi_status_entries) {  in hci_dma_process_ibi()
613 		ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr;  in hci_dma_process_ibi()
631 			ibi_size += chunks * rh->ibi_chunk_sz;  in hci_dma_process_ibi()
647 	deq_ptr %= rh->ibi_status_entries;  in hci_dma_process_ibi()
687 	ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr;  in hci_dma_process_ibi()
688 	ring_ibi_data = rh->ibi_data + ibi_data_offset;  in hci_dma_process_ibi()
689 	ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset;  in hci_dma_process_ibi()
690 	first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr)  in hci_dma_process_ibi()
691 			* rh->ibi_chunk_sz;  in hci_dma_process_ibi()
701 		ring_ibi_data = rh->ibi_data;  in hci_dma_process_ibi()
702 		ring_ibi_data_dma = rh->ibi_data_dma;  in hci_dma_process_ibi()
716 	spin_lock(&rh->lock);  in hci_dma_process_ibi()
721 	spin_unlock(&rh->lock);  in hci_dma_process_ibi()
724 	rh->ibi_chunk_ptr += ibi_chunks;  in hci_dma_process_ibi()
725 	rh->ibi_chunk_ptr %= rh->ibi_chunks_total;  in hci_dma_process_ibi()
738 		struct hci_rh_data *rh;  in hci_dma_irq_handler()  local
745 		rh = &rings->headers[i];  in hci_dma_irq_handler()
753 			hci_dma_process_ibi(hci, rh);  in hci_dma_irq_handler()
755 			hci_dma_xfer_done(hci, rh);  in hci_dma_irq_handler()
757 			complete(&rh->op_done);  in hci_dma_irq_handler()