Lines Matching refs:droq
94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) in octeon_droq_check_hw_for_pkts() argument
99 pkt_count = readl(droq->pkts_sent_reg); in octeon_droq_check_hw_for_pkts()
101 last_count = pkt_count - droq->pkt_count; in octeon_droq_check_hw_for_pkts()
102 droq->pkt_count = pkt_count; in octeon_droq_check_hw_for_pkts()
106 atomic_add(last_count, &droq->pkts_pending); in octeon_droq_check_hw_for_pkts()
111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) in octeon_droq_compute_max_packet_bufs() argument
120 droq->max_empty_descs = 0; in octeon_droq_compute_max_packet_bufs()
123 droq->max_empty_descs++; in octeon_droq_compute_max_packet_bufs()
124 count += droq->buffer_size; in octeon_droq_compute_max_packet_bufs()
127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; in octeon_droq_compute_max_packet_bufs()
130 static void octeon_droq_reset_indices(struct octeon_droq *droq) in octeon_droq_reset_indices() argument
132 droq->read_idx = 0; in octeon_droq_reset_indices()
133 droq->write_idx = 0; in octeon_droq_reset_indices()
134 droq->refill_idx = 0; in octeon_droq_reset_indices()
135 droq->refill_count = 0; in octeon_droq_reset_indices()
136 atomic_set(&droq->pkts_pending, 0); in octeon_droq_reset_indices()
141 struct octeon_droq *droq) in octeon_droq_destroy_ring_buffers() argument
146 for (i = 0; i < droq->max_count; i++) { in octeon_droq_destroy_ring_buffers()
147 pg_info = &droq->recv_buf_list[i].pg_info; in octeon_droq_destroy_ring_buffers()
157 recv_buffer_destroy(droq->recv_buf_list[i].buffer, in octeon_droq_destroy_ring_buffers()
160 droq->recv_buf_list[i].buffer = NULL; in octeon_droq_destroy_ring_buffers()
163 octeon_droq_reset_indices(droq); in octeon_droq_destroy_ring_buffers()
168 struct octeon_droq *droq) in octeon_droq_setup_ring_buffers() argument
172 struct octeon_droq_desc *desc_ring = droq->desc_ring; in octeon_droq_setup_ring_buffers()
174 for (i = 0; i < droq->max_count; i++) { in octeon_droq_setup_ring_buffers()
175 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); in octeon_droq_setup_ring_buffers()
180 droq->stats.rx_alloc_failure++; in octeon_droq_setup_ring_buffers()
184 droq->recv_buf_list[i].buffer = buf; in octeon_droq_setup_ring_buffers()
185 droq->recv_buf_list[i].data = get_rbd(buf); in octeon_droq_setup_ring_buffers()
188 lio_map_ring(droq->recv_buf_list[i].buffer); in octeon_droq_setup_ring_buffers()
191 octeon_droq_reset_indices(droq); in octeon_droq_setup_ring_buffers()
193 octeon_droq_compute_max_packet_bufs(droq); in octeon_droq_setup_ring_buffers()
200 struct octeon_droq *droq = oct->droq[q_no]; in octeon_delete_droq() local
204 octeon_droq_destroy_ring_buffers(oct, droq); in octeon_delete_droq()
205 vfree(droq->recv_buf_list); in octeon_delete_droq()
207 if (droq->desc_ring) in octeon_delete_droq()
208 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), in octeon_delete_droq()
209 droq->desc_ring, droq->desc_ring_dma); in octeon_delete_droq()
211 memset(droq, 0, OCT_DROQ_SIZE); in octeon_delete_droq()
213 vfree(oct->droq[q_no]); in octeon_delete_droq()
214 oct->droq[q_no] = NULL; in octeon_delete_droq()
226 struct octeon_droq *droq; in octeon_init_droq() local
233 droq = oct->droq[q_no]; in octeon_init_droq()
234 memset(droq, 0, OCT_DROQ_SIZE); in octeon_init_droq()
236 droq->oct_dev = oct; in octeon_init_droq()
237 droq->q_no = q_no; in octeon_init_droq()
239 droq->app_ctx = app_ctx; in octeon_init_droq()
241 droq->app_ctx = (void *)(size_t)q_no; in octeon_init_droq()
265 droq->max_count = c_num_descs; in octeon_init_droq()
266 droq->buffer_size = c_buf_size; in octeon_init_droq()
268 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; in octeon_init_droq()
269 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, in octeon_init_droq()
270 (dma_addr_t *)&droq->desc_ring_dma); in octeon_init_droq()
272 if (!droq->desc_ring) { in octeon_init_droq()
279 q_no, droq->desc_ring, droq->desc_ring_dma); in octeon_init_droq()
281 droq->max_count); in octeon_init_droq()
283 droq->recv_buf_list = (struct octeon_recv_buffer *) in octeon_init_droq()
284 vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), in octeon_init_droq()
286 if (!droq->recv_buf_list) in octeon_init_droq()
287 droq->recv_buf_list = (struct octeon_recv_buffer *) in octeon_init_droq()
288 vzalloc(array_size(droq->max_count, in octeon_init_droq()
290 if (!droq->recv_buf_list) { in octeon_init_droq()
295 if (octeon_droq_setup_ring_buffers(oct, droq)) in octeon_init_droq()
298 droq->pkts_per_intr = c_pkts_per_intr; in octeon_init_droq()
299 droq->refill_threshold = c_refill_threshold; in octeon_init_droq()
302 droq->max_empty_descs); in octeon_init_droq()
304 spin_lock_init(&droq->lock); in octeon_init_droq()
306 INIT_LIST_HEAD(&droq->dispatch_list); in octeon_init_droq()
341 struct octeon_droq *droq, in octeon_create_recv_info() argument
351 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; in octeon_create_recv_info()
368 pg_info = &droq->recv_buf_list[idx].pg_info; in octeon_create_recv_info()
378 droq->buffer_size) ? droq->buffer_size : bytes_left; in octeon_create_recv_info()
380 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; in octeon_create_recv_info()
381 droq->recv_buf_list[idx].buffer = NULL; in octeon_create_recv_info()
383 idx = incr_index(idx, 1, droq->max_count); in octeon_create_recv_info()
384 bytes_left -= droq->buffer_size; in octeon_create_recv_info()
396 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, in octeon_droq_refill_pullup_descs() argument
401 u32 refill_index = droq->refill_idx; in octeon_droq_refill_pullup_descs()
403 while (refill_index != droq->read_idx) { in octeon_droq_refill_pullup_descs()
404 if (droq->recv_buf_list[refill_index].buffer) { in octeon_droq_refill_pullup_descs()
405 droq->recv_buf_list[droq->refill_idx].buffer = in octeon_droq_refill_pullup_descs()
406 droq->recv_buf_list[refill_index].buffer; in octeon_droq_refill_pullup_descs()
407 droq->recv_buf_list[droq->refill_idx].data = in octeon_droq_refill_pullup_descs()
408 droq->recv_buf_list[refill_index].data; in octeon_droq_refill_pullup_descs()
409 desc_ring[droq->refill_idx].buffer_ptr = in octeon_droq_refill_pullup_descs()
411 droq->recv_buf_list[refill_index].buffer = NULL; in octeon_droq_refill_pullup_descs()
414 droq->refill_idx = incr_index(droq->refill_idx, in octeon_droq_refill_pullup_descs()
416 droq->max_count); in octeon_droq_refill_pullup_descs()
418 droq->refill_count--; in octeon_droq_refill_pullup_descs()
419 } while (droq->recv_buf_list[droq->refill_idx].buffer); in octeon_droq_refill_pullup_descs()
421 refill_index = incr_index(refill_index, 1, droq->max_count); in octeon_droq_refill_pullup_descs()
440 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) in octeon_droq_refill() argument
448 desc_ring = droq->desc_ring; in octeon_droq_refill()
450 while (droq->refill_count && (desc_refilled < droq->max_count)) { in octeon_droq_refill()
455 if (!droq->recv_buf_list[droq->refill_idx].buffer) { in octeon_droq_refill()
457 &droq->recv_buf_list[droq->refill_idx].pg_info; in octeon_droq_refill()
469 droq->stats.rx_alloc_failure++; in octeon_droq_refill()
472 droq->recv_buf_list[droq->refill_idx].buffer = in octeon_droq_refill()
476 data = get_rbd(droq->recv_buf_list in octeon_droq_refill()
477 [droq->refill_idx].buffer); in octeon_droq_refill()
480 droq->recv_buf_list[droq->refill_idx].data = data; in octeon_droq_refill()
482 desc_ring[droq->refill_idx].buffer_ptr = in octeon_droq_refill()
483 lio_map_ring(droq->recv_buf_list[ in octeon_droq_refill()
484 droq->refill_idx].buffer); in octeon_droq_refill()
486 droq->refill_idx = incr_index(droq->refill_idx, 1, in octeon_droq_refill()
487 droq->max_count); in octeon_droq_refill()
489 droq->refill_count--; in octeon_droq_refill()
492 if (droq->refill_count) in octeon_droq_refill()
494 octeon_droq_refill_pullup_descs(droq, desc_ring); in octeon_droq_refill()
508 void octeon_droq_check_oom(struct octeon_droq *droq) in octeon_droq_check_oom() argument
511 struct octeon_device *oct = droq->oct_dev; in octeon_droq_check_oom()
513 if (readl(droq->pkts_credit_reg) <= CN23XX_SLI_DEF_BP) { in octeon_droq_check_oom()
514 spin_lock_bh(&droq->lock); in octeon_droq_check_oom()
515 desc_refilled = octeon_droq_refill(oct, droq); in octeon_droq_check_oom()
522 writel(desc_refilled, droq->pkts_credit_reg); in octeon_droq_check_oom()
526 spin_unlock_bh(&droq->lock); in octeon_droq_check_oom()
538 struct octeon_droq *droq, in octeon_droq_dispatch_pkt() argument
546 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); in octeon_droq_dispatch_pkt()
551 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); in octeon_droq_dispatch_pkt()
559 &droq->dispatch_list); in octeon_droq_dispatch_pkt()
561 droq->stats.dropped_nomem++; in octeon_droq_dispatch_pkt()
567 droq->stats.dropped_nodispatch++; in octeon_droq_dispatch_pkt()
574 struct octeon_droq *droq, in octeon_droq_drop_packets() argument
582 droq->recv_buf_list[droq->read_idx].data; in octeon_droq_drop_packets()
587 droq->stats.bytes_received += info->length; in octeon_droq_drop_packets()
588 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, in octeon_droq_drop_packets()
595 droq->read_idx = incr_index(droq->read_idx, buf_cnt, in octeon_droq_drop_packets()
596 droq->max_count); in octeon_droq_drop_packets()
597 droq->refill_count += buf_cnt; in octeon_droq_drop_packets()
603 struct octeon_droq *droq, in octeon_droq_fast_process_packets() argument
619 droq->recv_buf_list[droq->read_idx].data; in octeon_droq_fast_process_packets()
625 droq->q_no, droq->read_idx, pkt_count); in octeon_droq_fast_process_packets()
641 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); in octeon_droq_fast_process_packets()
642 droq->read_idx = incr_index(droq->read_idx, in octeon_droq_fast_process_packets()
643 buf_cnt, droq->max_count); in octeon_droq_fast_process_packets()
644 droq->refill_count += buf_cnt; in octeon_droq_fast_process_packets()
646 if (info->length <= droq->buffer_size) { in octeon_droq_fast_process_packets()
648 nicbuf = droq->recv_buf_list[ in octeon_droq_fast_process_packets()
649 droq->read_idx].buffer; in octeon_droq_fast_process_packets()
650 pg_info = &droq->recv_buf_list[ in octeon_droq_fast_process_packets()
651 droq->read_idx].pg_info; in octeon_droq_fast_process_packets()
654 droq->recv_buf_list[droq->read_idx].buffer = in octeon_droq_fast_process_packets()
657 droq->read_idx = incr_index(droq->read_idx, 1, in octeon_droq_fast_process_packets()
658 droq->max_count); in octeon_droq_fast_process_packets()
659 droq->refill_count++; in octeon_droq_fast_process_packets()
668 int cpy_len, idx = droq->read_idx; in octeon_droq_fast_process_packets()
670 cpy_len = ((pkt_len + droq->buffer_size) in octeon_droq_fast_process_packets()
673 droq->buffer_size; in octeon_droq_fast_process_packets()
676 octeon_fast_packet_next(droq, in octeon_droq_fast_process_packets()
680 buf = droq->recv_buf_list[ in octeon_droq_fast_process_packets()
683 droq->recv_buf_list[idx].buffer in octeon_droq_fast_process_packets()
686 droq->stats.rx_alloc_failure++; in octeon_droq_fast_process_packets()
690 droq->read_idx = in octeon_droq_fast_process_packets()
691 incr_index(droq->read_idx, 1, in octeon_droq_fast_process_packets()
692 droq->max_count); in octeon_droq_fast_process_packets()
693 droq->refill_count++; in octeon_droq_fast_process_packets()
698 if (droq->ops.fptr) { in octeon_droq_fast_process_packets()
699 droq->ops.fptr(oct->octeon_id, in octeon_droq_fast_process_packets()
701 rh, &droq->napi, in octeon_droq_fast_process_packets()
702 droq->ops.farg); in octeon_droq_fast_process_packets()
709 if (droq->refill_count >= droq->refill_threshold) { in octeon_droq_fast_process_packets()
710 int desc_refilled = octeon_droq_refill(oct, droq); in octeon_droq_fast_process_packets()
717 writel((desc_refilled), droq->pkts_credit_reg); in octeon_droq_fast_process_packets()
725 droq->stats.pkts_received += pkt; in octeon_droq_fast_process_packets()
726 droq->stats.bytes_received += total_len; in octeon_droq_fast_process_packets()
728 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { in octeon_droq_fast_process_packets()
729 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); in octeon_droq_fast_process_packets()
731 droq->stats.dropped_toomany += (pkts_to_process - pkt); in octeon_droq_fast_process_packets()
740 struct octeon_droq *droq, in octeon_droq_process_packets() argument
747 spin_lock(&droq->lock); in octeon_droq_process_packets()
749 octeon_droq_check_hw_for_pkts(droq); in octeon_droq_process_packets()
750 pkt_count = atomic_read(&droq->pkts_pending); in octeon_droq_process_packets()
753 spin_unlock(&droq->lock); in octeon_droq_process_packets()
760 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); in octeon_droq_process_packets()
762 atomic_sub(pkts_processed, &droq->pkts_pending); in octeon_droq_process_packets()
765 spin_unlock(&droq->lock); in octeon_droq_process_packets()
767 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { in octeon_droq_process_packets()
779 if (atomic_read(&droq->pkts_pending)) in octeon_droq_process_packets()
792 struct octeon_droq *droq, u32 budget) in octeon_droq_process_poll_pkts() argument
798 if (budget > droq->max_count) in octeon_droq_process_poll_pkts()
799 budget = droq->max_count; in octeon_droq_process_poll_pkts()
801 spin_lock(&droq->lock); in octeon_droq_process_poll_pkts()
804 octeon_droq_check_hw_for_pkts(droq); in octeon_droq_process_poll_pkts()
807 (u32)(atomic_read(&droq->pkts_pending))); in octeon_droq_process_poll_pkts()
813 octeon_droq_fast_process_packets(oct, droq, in octeon_droq_process_poll_pkts()
816 atomic_sub(pkts_processed, &droq->pkts_pending); in octeon_droq_process_poll_pkts()
821 spin_unlock(&droq->lock); in octeon_droq_process_poll_pkts()
823 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { in octeon_droq_process_poll_pkts()
865 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); in octeon_enable_irq()
869 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); in octeon_enable_irq()
882 struct octeon_droq *droq; in octeon_register_droq_ops() local
903 droq = oct->droq[q_no]; in octeon_register_droq_ops()
905 spin_lock_irqsave(&droq->lock, flags); in octeon_register_droq_ops()
907 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); in octeon_register_droq_ops()
909 spin_unlock_irqrestore(&droq->lock, flags); in octeon_register_droq_ops()
917 struct octeon_droq *droq; in octeon_unregister_droq_ops() local
931 droq = oct->droq[q_no]; in octeon_unregister_droq_ops()
933 if (!droq) { in octeon_unregister_droq_ops()
939 spin_lock_irqsave(&droq->lock, flags); in octeon_unregister_droq_ops()
941 droq->ops.fptr = NULL; in octeon_unregister_droq_ops()
942 droq->ops.farg = NULL; in octeon_unregister_droq_ops()
943 droq->ops.drop_on_max = 0; in octeon_unregister_droq_ops()
945 spin_unlock_irqrestore(&droq->lock, flags); in octeon_unregister_droq_ops()
954 struct octeon_droq *droq; in octeon_create_droq() local
957 if (oct->droq[q_no]) { in octeon_create_droq()
964 droq = vmalloc_node(sizeof(*droq), numa_node); in octeon_create_droq()
965 if (!droq) in octeon_create_droq()
966 droq = vmalloc(sizeof(*droq)); in octeon_create_droq()
967 if (!droq) in octeon_create_droq()
970 memset(droq, 0, sizeof(struct octeon_droq)); in octeon_create_droq()
974 oct->droq[q_no] = droq; in octeon_create_droq()
978 vfree(oct->droq[q_no]); in octeon_create_droq()
979 oct->droq[q_no] = NULL; in octeon_create_droq()