57 static inline struct dpaa2_io *service_select_by_cpu(struct dpaa2_io *d,  in service_select_by_cpu()  argument
60 	if (d)  in service_select_by_cpu()
61 		return d;  in service_select_by_cpu()
77 static inline struct dpaa2_io *service_select(struct dpaa2_io *d)  in service_select()  argument
79 	if (d)  in service_select()
80 		return d;  in service_select()
82 	d = service_select_by_cpu(d, -1);  in service_select()
83 	if (d)  in service_select()
84 		return d;  in service_select()
87 	d = list_entry(dpio_list.next, struct dpaa2_io, node);  in service_select()
88 	list_del(&d->node);  in service_select()
89 	list_add_tail(&d->node, &dpio_list);  in service_select()
92 	return d;  in service_select()
117 	struct dpaa2_io *d = container_of(dim, struct dpaa2_io, rx_dim);  in dpaa2_io_dim_work()  local
119 	dpaa2_io_set_irq_coalescing(d, moder.usec);  in dpaa2_io_dim_work()
206 void dpaa2_io_down(struct dpaa2_io *d)  in dpaa2_io_down()  argument
209 	dpio_by_cpu[d->dpio_desc.cpu] = NULL;  in dpaa2_io_down()
210 	list_del(&d->node);  in dpaa2_io_down()
213 	kfree(d);  in dpaa2_io_down()
271 int dpaa2_io_get_cpu(struct dpaa2_io *d)  in dpaa2_io_get_cpu()  argument
273 	return d->dpio_desc.cpu;  in dpaa2_io_get_cpu()
296 int dpaa2_io_service_register(struct dpaa2_io *d,  in dpaa2_io_service_register()  argument
303 	d = service_select_by_cpu(d, ctx->desired_cpu);  in dpaa2_io_service_register()
304 	if (!d)  in dpaa2_io_service_register()
307 	link = device_link_add(dev, d->dev, DL_FLAG_AUTOREMOVE_CONSUMER);  in dpaa2_io_service_register()
311 	ctx->dpio_id = d->dpio_desc.dpio_id;  in dpaa2_io_service_register()
313 	ctx->dpio_private = d;  in dpaa2_io_service_register()
314 	spin_lock_irqsave(&d->lock_notifications, irqflags);  in dpaa2_io_service_register()
315 	list_add(&ctx->node, &d->notifications);  in dpaa2_io_service_register()
316 	spin_unlock_irqrestore(&d->lock_notifications, irqflags);  in dpaa2_io_service_register()
320 		return qbman_swp_CDAN_set_context_enable(d->swp,  in dpaa2_io_service_register()
340 	struct dpaa2_io *d = ctx->dpio_private;  in dpaa2_io_service_deregister()  local
344 		qbman_swp_CDAN_disable(d->swp, (u16)ctx->id);  in dpaa2_io_service_deregister()
346 	spin_lock_irqsave(&d->lock_notifications, irqflags);  in dpaa2_io_service_deregister()
348 	spin_unlock_irqrestore(&d->lock_notifications, irqflags);  in dpaa2_io_service_deregister()
366 int dpaa2_io_service_rearm(struct dpaa2_io *d,  in dpaa2_io_service_rearm()  argument
372 	d = service_select_by_cpu(d, ctx->desired_cpu);  in dpaa2_io_service_rearm()
373 	if (!unlikely(d))  in dpaa2_io_service_rearm()
376 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_rearm()
378 		err = qbman_swp_CDAN_enable(d->swp, (u16)ctx->id);  in dpaa2_io_service_rearm()
380 		err = qbman_swp_fq_schedule(d->swp, ctx->id);  in dpaa2_io_service_rearm()
381 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_rearm()
395 int dpaa2_io_service_pull_fq(struct dpaa2_io *d, u32 fqid,  in dpaa2_io_service_pull_fq()  argument
406 	d = service_select(d);  in dpaa2_io_service_pull_fq()
407 	if (!d)  in dpaa2_io_service_pull_fq()
409 	s->swp = d->swp;  in dpaa2_io_service_pull_fq()
410 	err = qbman_swp_pull(d->swp, &pd);  in dpaa2_io_service_pull_fq()
426 int dpaa2_io_service_pull_channel(struct dpaa2_io *d, u32 channelid,  in dpaa2_io_service_pull_channel()  argument
437 	d = service_select(d);  in dpaa2_io_service_pull_channel()
438 	if (!d)  in dpaa2_io_service_pull_channel()
441 	s->swp = d->swp;  in dpaa2_io_service_pull_channel()
442 	err = qbman_swp_pull(d->swp, &pd);  in dpaa2_io_service_pull_channel()
459 int dpaa2_io_service_enqueue_fq(struct dpaa2_io *d,  in dpaa2_io_service_enqueue_fq()  argument
465 	d = service_select(d);  in dpaa2_io_service_enqueue_fq()
466 	if (!d)  in dpaa2_io_service_enqueue_fq()
473 	return qbman_swp_enqueue(d->swp, &ed, fd);  in dpaa2_io_service_enqueue_fq()
488 int dpaa2_io_service_enqueue_multiple_fq(struct dpaa2_io *d,  in dpaa2_io_service_enqueue_multiple_fq()  argument
495 	d = service_select(d);  in dpaa2_io_service_enqueue_multiple_fq()
496 	if (!d)  in dpaa2_io_service_enqueue_multiple_fq()
503 	return qbman_swp_enqueue_multiple(d->swp, &ed, fd, NULL, nb);  in dpaa2_io_service_enqueue_multiple_fq()
518 int dpaa2_io_service_enqueue_multiple_desc_fq(struct dpaa2_io *d,  in dpaa2_io_service_enqueue_multiple_desc_fq()  argument
530 	d = service_select(d);  in dpaa2_io_service_enqueue_multiple_desc_fq()
531 	if (!d) {  in dpaa2_io_service_enqueue_multiple_desc_fq()
542 	ret = qbman_swp_enqueue_multiple_desc(d->swp, &ed[0], fd, nb);  in dpaa2_io_service_enqueue_multiple_desc_fq()
560 int dpaa2_io_service_enqueue_qd(struct dpaa2_io *d,  in dpaa2_io_service_enqueue_qd()  argument
566 	d = service_select(d);  in dpaa2_io_service_enqueue_qd()
567 	if (!d)  in dpaa2_io_service_enqueue_qd()
574 	return qbman_swp_enqueue(d->swp, &ed, fd);  in dpaa2_io_service_enqueue_qd()
587 int dpaa2_io_service_release(struct dpaa2_io *d,  in dpaa2_io_service_release()  argument
594 	d = service_select(d);  in dpaa2_io_service_release()
595 	if (!d)  in dpaa2_io_service_release()
601 	return qbman_swp_release(d->swp, &rd, buffers, num_buffers);  in dpaa2_io_service_release()
616 int dpaa2_io_service_acquire(struct dpaa2_io *d,  in dpaa2_io_service_acquire()  argument
624 	d = service_select(d);  in dpaa2_io_service_acquire()
625 	if (!d)  in dpaa2_io_service_acquire()
628 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_acquire()
629 	err = qbman_swp_acquire(d->swp, bpid, buffers, num_buffers);  in dpaa2_io_service_acquire()
630 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_acquire()
765 int dpaa2_io_query_fq_count(struct dpaa2_io *d, u32 fqid,  in dpaa2_io_query_fq_count()  argument
773 	d = service_select(d);  in dpaa2_io_query_fq_count()
774 	if (!d)  in dpaa2_io_query_fq_count()
777 	swp = d->swp;  in dpaa2_io_query_fq_count()
778 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_fq_count()
780 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_fq_count()
799 int dpaa2_io_query_bp_count(struct dpaa2_io *d, u16 bpid, u32 *num)  in dpaa2_io_query_bp_count()  argument
806 	d = service_select(d);  in dpaa2_io_query_bp_count()
807 	if (!d)  in dpaa2_io_query_bp_count()
810 	swp = d->swp;  in dpaa2_io_query_bp_count()
811 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_bp_count()
813 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_bp_count()
828 int dpaa2_io_set_irq_coalescing(struct dpaa2_io *d, u32 irq_holdoff)  in dpaa2_io_set_irq_coalescing()  argument
830 	struct qbman_swp *swp = d->swp;  in dpaa2_io_set_irq_coalescing()
842 void dpaa2_io_get_irq_coalescing(struct dpaa2_io *d, u32 *irq_holdoff)  in dpaa2_io_get_irq_coalescing()  argument
844 	struct qbman_swp *swp = d->swp;  in dpaa2_io_get_irq_coalescing()
855 void dpaa2_io_set_adaptive_coalescing(struct dpaa2_io *d,  in dpaa2_io_set_adaptive_coalescing()  argument
858 	d->swp->use_adaptive_rx_coalesce = use_adaptive_rx_coalesce;  in dpaa2_io_set_adaptive_coalescing()
869 int dpaa2_io_get_adaptive_coalescing(struct dpaa2_io *d)  in dpaa2_io_get_adaptive_coalescing()  argument
871 	return d->swp->use_adaptive_rx_coalesce;  in dpaa2_io_get_adaptive_coalescing()
881 void dpaa2_io_update_net_dim(struct dpaa2_io *d, __u64 frames, __u64 bytes)  in dpaa2_io_update_net_dim()  argument
885 	if (!d->swp->use_adaptive_rx_coalesce)  in dpaa2_io_update_net_dim()
888 	spin_lock(&d->dim_lock);  in dpaa2_io_update_net_dim()
890 	d->bytes += bytes;  in dpaa2_io_update_net_dim()
891 	d->frames += frames;  in dpaa2_io_update_net_dim()
893 	dim_update_sample(d->event_ctr, d->frames, d->bytes, &dim_sample);  in dpaa2_io_update_net_dim()
894 	net_dim(&d->rx_dim, dim_sample);  in dpaa2_io_update_net_dim()
896 	spin_unlock(&d->dim_lock);  in dpaa2_io_update_net_dim()