Lines Matching refs:droq
94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) in octeon_droq_check_hw_for_pkts() argument
99 pkt_count = readl(droq->pkts_sent_reg); in octeon_droq_check_hw_for_pkts()
101 last_count = pkt_count - droq->pkt_count; in octeon_droq_check_hw_for_pkts()
102 droq->pkt_count = pkt_count; in octeon_droq_check_hw_for_pkts()
106 atomic_add(last_count, &droq->pkts_pending); in octeon_droq_check_hw_for_pkts()
111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) in octeon_droq_compute_max_packet_bufs() argument
120 droq->max_empty_descs = 0; in octeon_droq_compute_max_packet_bufs()
123 droq->max_empty_descs++; in octeon_droq_compute_max_packet_bufs()
124 count += droq->buffer_size; in octeon_droq_compute_max_packet_bufs()
127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; in octeon_droq_compute_max_packet_bufs()
130 static void octeon_droq_reset_indices(struct octeon_droq *droq) in octeon_droq_reset_indices() argument
132 droq->read_idx = 0; in octeon_droq_reset_indices()
133 droq->write_idx = 0; in octeon_droq_reset_indices()
134 droq->refill_idx = 0; in octeon_droq_reset_indices()
135 droq->refill_count = 0; in octeon_droq_reset_indices()
136 atomic_set(&droq->pkts_pending, 0); in octeon_droq_reset_indices()
141 struct octeon_droq *droq) in octeon_droq_destroy_ring_buffers() argument
146 for (i = 0; i < droq->max_count; i++) { in octeon_droq_destroy_ring_buffers()
147 pg_info = &droq->recv_buf_list[i].pg_info; in octeon_droq_destroy_ring_buffers()
157 recv_buffer_destroy(droq->recv_buf_list[i].buffer, in octeon_droq_destroy_ring_buffers()
160 droq->recv_buf_list[i].buffer = NULL; in octeon_droq_destroy_ring_buffers()
163 octeon_droq_reset_indices(droq); in octeon_droq_destroy_ring_buffers()
168 struct octeon_droq *droq) in octeon_droq_setup_ring_buffers() argument
172 struct octeon_droq_desc *desc_ring = droq->desc_ring; in octeon_droq_setup_ring_buffers()
174 for (i = 0; i < droq->max_count; i++) { in octeon_droq_setup_ring_buffers()
175 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); in octeon_droq_setup_ring_buffers()
180 droq->stats.rx_alloc_failure++; in octeon_droq_setup_ring_buffers()
184 droq->recv_buf_list[i].buffer = buf; in octeon_droq_setup_ring_buffers()
185 droq->recv_buf_list[i].data = get_rbd(buf); in octeon_droq_setup_ring_buffers()
188 lio_map_ring(droq->recv_buf_list[i].buffer); in octeon_droq_setup_ring_buffers()
191 octeon_droq_reset_indices(droq); in octeon_droq_setup_ring_buffers()
193 octeon_droq_compute_max_packet_bufs(droq); in octeon_droq_setup_ring_buffers()
200 struct octeon_droq *droq = oct->droq[q_no]; in octeon_delete_droq() local
204 octeon_droq_destroy_ring_buffers(oct, droq); in octeon_delete_droq()
205 vfree(droq->recv_buf_list); in octeon_delete_droq()
207 if (droq->desc_ring) in octeon_delete_droq()
208 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), in octeon_delete_droq()
209 droq->desc_ring, droq->desc_ring_dma); in octeon_delete_droq()
211 memset(droq, 0, OCT_DROQ_SIZE); in octeon_delete_droq()
213 vfree(oct->droq[q_no]); in octeon_delete_droq()
214 oct->droq[q_no] = NULL; in octeon_delete_droq()
226 struct octeon_droq *droq; in octeon_init_droq() local
233 droq = oct->droq[q_no]; in octeon_init_droq()
234 memset(droq, 0, OCT_DROQ_SIZE); in octeon_init_droq()
236 droq->oct_dev = oct; in octeon_init_droq()
237 droq->q_no = q_no; in octeon_init_droq()
239 droq->app_ctx = app_ctx; in octeon_init_droq()
241 droq->app_ctx = (void *)(size_t)q_no; in octeon_init_droq()
265 droq->max_count = c_num_descs; in octeon_init_droq()
266 droq->buffer_size = c_buf_size; in octeon_init_droq()
268 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; in octeon_init_droq()
269 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, in octeon_init_droq()
270 (dma_addr_t *)&droq->desc_ring_dma); in octeon_init_droq()
272 if (!droq->desc_ring) { in octeon_init_droq()
279 q_no, droq->desc_ring, droq->desc_ring_dma); in octeon_init_droq()
281 droq->max_count); in octeon_init_droq()
283 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), in octeon_init_droq()
285 if (!droq->recv_buf_list) in octeon_init_droq()
286 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE)); in octeon_init_droq()
287 if (!droq->recv_buf_list) { in octeon_init_droq()
292 if (octeon_droq_setup_ring_buffers(oct, droq)) in octeon_init_droq()
295 droq->pkts_per_intr = c_pkts_per_intr; in octeon_init_droq()
296 droq->refill_threshold = c_refill_threshold; in octeon_init_droq()
299 droq->max_empty_descs); in octeon_init_droq()
301 INIT_LIST_HEAD(&droq->dispatch_list); in octeon_init_droq()
334 struct octeon_droq *droq, in octeon_create_recv_info() argument
344 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; in octeon_create_recv_info()
361 pg_info = &droq->recv_buf_list[idx].pg_info; in octeon_create_recv_info()
371 droq->buffer_size) ? droq->buffer_size : bytes_left; in octeon_create_recv_info()
373 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; in octeon_create_recv_info()
374 droq->recv_buf_list[idx].buffer = NULL; in octeon_create_recv_info()
376 idx = incr_index(idx, 1, droq->max_count); in octeon_create_recv_info()
377 bytes_left -= droq->buffer_size; in octeon_create_recv_info()
389 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, in octeon_droq_refill_pullup_descs() argument
394 u32 refill_index = droq->refill_idx; in octeon_droq_refill_pullup_descs()
396 while (refill_index != droq->read_idx) { in octeon_droq_refill_pullup_descs()
397 if (droq->recv_buf_list[refill_index].buffer) { in octeon_droq_refill_pullup_descs()
398 droq->recv_buf_list[droq->refill_idx].buffer = in octeon_droq_refill_pullup_descs()
399 droq->recv_buf_list[refill_index].buffer; in octeon_droq_refill_pullup_descs()
400 droq->recv_buf_list[droq->refill_idx].data = in octeon_droq_refill_pullup_descs()
401 droq->recv_buf_list[refill_index].data; in octeon_droq_refill_pullup_descs()
402 desc_ring[droq->refill_idx].buffer_ptr = in octeon_droq_refill_pullup_descs()
404 droq->recv_buf_list[refill_index].buffer = NULL; in octeon_droq_refill_pullup_descs()
407 droq->refill_idx = incr_index(droq->refill_idx, in octeon_droq_refill_pullup_descs()
409 droq->max_count); in octeon_droq_refill_pullup_descs()
411 droq->refill_count--; in octeon_droq_refill_pullup_descs()
412 } while (droq->recv_buf_list[droq->refill_idx].buffer); in octeon_droq_refill_pullup_descs()
414 refill_index = incr_index(refill_index, 1, droq->max_count); in octeon_droq_refill_pullup_descs()
431 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) in octeon_droq_refill() argument
439 desc_ring = droq->desc_ring; in octeon_droq_refill()
441 while (droq->refill_count && (desc_refilled < droq->max_count)) { in octeon_droq_refill()
445 if (!droq->recv_buf_list[droq->refill_idx].buffer) { in octeon_droq_refill()
447 &droq->recv_buf_list[droq->refill_idx].pg_info; in octeon_droq_refill()
459 droq->stats.rx_alloc_failure++; in octeon_droq_refill()
462 droq->recv_buf_list[droq->refill_idx].buffer = in octeon_droq_refill()
466 data = get_rbd(droq->recv_buf_list in octeon_droq_refill()
467 [droq->refill_idx].buffer); in octeon_droq_refill()
470 droq->recv_buf_list[droq->refill_idx].data = data; in octeon_droq_refill()
472 desc_ring[droq->refill_idx].buffer_ptr = in octeon_droq_refill()
473 lio_map_ring(droq->recv_buf_list[ in octeon_droq_refill()
474 droq->refill_idx].buffer); in octeon_droq_refill()
476 droq->refill_idx = incr_index(droq->refill_idx, 1, in octeon_droq_refill()
477 droq->max_count); in octeon_droq_refill()
479 droq->refill_count--; in octeon_droq_refill()
482 if (droq->refill_count) in octeon_droq_refill()
484 octeon_droq_refill_pullup_descs(droq, desc_ring); in octeon_droq_refill()
498 int octeon_retry_droq_refill(struct octeon_droq *droq) in octeon_retry_droq_refill() argument
500 struct octeon_device *oct = droq->oct_dev; in octeon_retry_droq_refill()
504 pkts_credit = readl(droq->pkts_credit_reg); in octeon_retry_droq_refill()
505 desc_refilled = octeon_droq_refill(oct, droq); in octeon_retry_droq_refill()
512 writel(desc_refilled, droq->pkts_credit_reg); in octeon_retry_droq_refill()
529 struct octeon_droq *droq, in octeon_droq_dispatch_pkt() argument
537 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); in octeon_droq_dispatch_pkt()
542 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); in octeon_droq_dispatch_pkt()
550 &droq->dispatch_list); in octeon_droq_dispatch_pkt()
552 droq->stats.dropped_nomem++; in octeon_droq_dispatch_pkt()
558 droq->stats.dropped_nodispatch++; in octeon_droq_dispatch_pkt()
565 struct octeon_droq *droq, in octeon_droq_drop_packets() argument
573 droq->recv_buf_list[droq->read_idx].data; in octeon_droq_drop_packets()
578 droq->stats.bytes_received += info->length; in octeon_droq_drop_packets()
579 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, in octeon_droq_drop_packets()
586 droq->read_idx = incr_index(droq->read_idx, buf_cnt, in octeon_droq_drop_packets()
587 droq->max_count); in octeon_droq_drop_packets()
588 droq->refill_count += buf_cnt; in octeon_droq_drop_packets()
594 struct octeon_droq *droq, in octeon_droq_fast_process_packets() argument
610 droq->recv_buf_list[droq->read_idx].data; in octeon_droq_fast_process_packets()
616 droq->q_no, droq->read_idx, pkt_count); in octeon_droq_fast_process_packets()
632 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); in octeon_droq_fast_process_packets()
633 droq->read_idx = incr_index(droq->read_idx, in octeon_droq_fast_process_packets()
634 buf_cnt, droq->max_count); in octeon_droq_fast_process_packets()
635 droq->refill_count += buf_cnt; in octeon_droq_fast_process_packets()
637 if (info->length <= droq->buffer_size) { in octeon_droq_fast_process_packets()
639 nicbuf = droq->recv_buf_list[ in octeon_droq_fast_process_packets()
640 droq->read_idx].buffer; in octeon_droq_fast_process_packets()
641 pg_info = &droq->recv_buf_list[ in octeon_droq_fast_process_packets()
642 droq->read_idx].pg_info; in octeon_droq_fast_process_packets()
645 droq->recv_buf_list[droq->read_idx].buffer = in octeon_droq_fast_process_packets()
648 droq->read_idx = incr_index(droq->read_idx, 1, in octeon_droq_fast_process_packets()
649 droq->max_count); in octeon_droq_fast_process_packets()
650 droq->refill_count++; in octeon_droq_fast_process_packets()
659 int cpy_len, idx = droq->read_idx; in octeon_droq_fast_process_packets()
661 cpy_len = ((pkt_len + droq->buffer_size) in octeon_droq_fast_process_packets()
664 droq->buffer_size; in octeon_droq_fast_process_packets()
667 octeon_fast_packet_next(droq, in octeon_droq_fast_process_packets()
671 buf = droq->recv_buf_list[ in octeon_droq_fast_process_packets()
674 droq->recv_buf_list[idx].buffer in octeon_droq_fast_process_packets()
677 droq->stats.rx_alloc_failure++; in octeon_droq_fast_process_packets()
681 droq->read_idx = in octeon_droq_fast_process_packets()
682 incr_index(droq->read_idx, 1, in octeon_droq_fast_process_packets()
683 droq->max_count); in octeon_droq_fast_process_packets()
684 droq->refill_count++; in octeon_droq_fast_process_packets()
689 if (droq->ops.fptr) { in octeon_droq_fast_process_packets()
690 droq->ops.fptr(oct->octeon_id, in octeon_droq_fast_process_packets()
692 rh, &droq->napi, in octeon_droq_fast_process_packets()
693 droq->ops.farg); in octeon_droq_fast_process_packets()
700 if (droq->refill_count >= droq->refill_threshold) { in octeon_droq_fast_process_packets()
701 int desc_refilled = octeon_droq_refill(oct, droq); in octeon_droq_fast_process_packets()
709 writel(desc_refilled, droq->pkts_credit_reg); in octeon_droq_fast_process_packets()
715 droq->stats.pkts_received += pkt; in octeon_droq_fast_process_packets()
716 droq->stats.bytes_received += total_len; in octeon_droq_fast_process_packets()
719 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { in octeon_droq_fast_process_packets()
720 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); in octeon_droq_fast_process_packets()
722 droq->stats.dropped_toomany += (pkts_to_process - pkt); in octeon_droq_fast_process_packets()
726 atomic_sub(retval, &droq->pkts_pending); in octeon_droq_fast_process_packets()
728 if (droq->refill_count >= droq->refill_threshold && in octeon_droq_fast_process_packets()
729 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) { in octeon_droq_fast_process_packets()
730 octeon_droq_check_hw_for_pkts(droq); in octeon_droq_fast_process_packets()
733 if (!atomic_read(&droq->pkts_pending)) in octeon_droq_fast_process_packets()
734 octeon_schedule_rxq_oom_work(oct, droq); in octeon_droq_fast_process_packets()
742 struct octeon_droq *droq, in octeon_droq_process_packets() argument
748 octeon_droq_check_hw_for_pkts(droq); in octeon_droq_process_packets()
749 pkt_count = atomic_read(&droq->pkts_pending); in octeon_droq_process_packets()
757 octeon_droq_fast_process_packets(oct, droq, pkt_count); in octeon_droq_process_packets()
759 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { in octeon_droq_process_packets()
771 if (atomic_read(&droq->pkts_pending)) in octeon_droq_process_packets()
784 struct octeon_droq *droq, u32 budget) in octeon_droq_process_poll_pkts() argument
790 if (budget > droq->max_count) in octeon_droq_process_poll_pkts()
791 budget = droq->max_count; in octeon_droq_process_poll_pkts()
794 octeon_droq_check_hw_for_pkts(droq); in octeon_droq_process_poll_pkts()
797 (u32)(atomic_read(&droq->pkts_pending))); in octeon_droq_process_poll_pkts()
803 octeon_droq_fast_process_packets(oct, droq, in octeon_droq_process_poll_pkts()
809 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { in octeon_droq_process_poll_pkts()
851 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); in octeon_enable_irq()
855 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); in octeon_enable_irq()
869 struct octeon_droq *droq; in octeon_register_droq_ops() local
888 droq = oct->droq[q_no]; in octeon_register_droq_ops()
889 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); in octeon_register_droq_ops()
897 struct octeon_droq *droq; in octeon_unregister_droq_ops() local
910 droq = oct->droq[q_no]; in octeon_unregister_droq_ops()
912 if (!droq) { in octeon_unregister_droq_ops()
918 droq->ops.fptr = NULL; in octeon_unregister_droq_ops()
919 droq->ops.farg = NULL; in octeon_unregister_droq_ops()
920 droq->ops.drop_on_max = 0; in octeon_unregister_droq_ops()
929 struct octeon_droq *droq; in octeon_create_droq() local
932 if (oct->droq[q_no]) { in octeon_create_droq()
939 droq = vmalloc_node(sizeof(*droq), numa_node); in octeon_create_droq()
940 if (!droq) in octeon_create_droq()
941 droq = vmalloc(sizeof(*droq)); in octeon_create_droq()
942 if (!droq) in octeon_create_droq()
945 memset(droq, 0, sizeof(struct octeon_droq)); in octeon_create_droq()
949 oct->droq[q_no] = droq; in octeon_create_droq()
953 vfree(oct->droq[q_no]); in octeon_create_droq()
954 oct->droq[q_no] = NULL; in octeon_create_droq()