Lines Matching full:tx
16 static bool gve_has_pending_packet(struct gve_tx_ring *tx) in gve_has_pending_packet() argument
18 /* Check TX path's list. */ in gve_has_pending_packet()
19 if (tx->dqo_tx.free_pending_packets != -1) in gve_has_pending_packet()
23 if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1) in gve_has_pending_packet()
30 gve_alloc_pending_packet(struct gve_tx_ring *tx) in gve_alloc_pending_packet() argument
35 index = tx->dqo_tx.free_pending_packets; in gve_alloc_pending_packet()
41 tx->dqo_tx.free_pending_packets = in gve_alloc_pending_packet()
42 atomic_xchg(&tx->dqo_compl.free_pending_packets, -1); in gve_alloc_pending_packet()
43 index = tx->dqo_tx.free_pending_packets; in gve_alloc_pending_packet()
49 pending_packet = &tx->dqo.pending_packets[index]; in gve_alloc_pending_packet()
52 tx->dqo_tx.free_pending_packets = pending_packet->next; in gve_alloc_pending_packet()
59 gve_free_pending_packet(struct gve_tx_ring *tx, in gve_free_pending_packet() argument
62 s16 index = pending_packet - tx->dqo.pending_packets; in gve_free_pending_packet()
66 s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets); in gve_free_pending_packet()
69 if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets, in gve_free_pending_packet()
76 /* gve_tx_free_desc - Cleans up all pending tx requests and buffers.
78 static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) in gve_tx_clean_pending_packets() argument
82 for (i = 0; i < tx->dqo.num_pending_packets; i++) { in gve_tx_clean_pending_packets()
84 &tx->dqo.pending_packets[i]; in gve_tx_clean_pending_packets()
89 dma_unmap_single(tx->dev, in gve_tx_clean_pending_packets()
94 dma_unmap_page(tx->dev, in gve_tx_clean_pending_packets()
109 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_free_ring_dqo() local
115 if (tx->q_resources) { in gve_tx_free_ring_dqo()
116 dma_free_coherent(hdev, sizeof(*tx->q_resources), in gve_tx_free_ring_dqo()
117 tx->q_resources, tx->q_resources_bus); in gve_tx_free_ring_dqo()
118 tx->q_resources = NULL; in gve_tx_free_ring_dqo()
121 if (tx->dqo.compl_ring) { in gve_tx_free_ring_dqo()
122 bytes = sizeof(tx->dqo.compl_ring[0]) * in gve_tx_free_ring_dqo()
123 (tx->dqo.complq_mask + 1); in gve_tx_free_ring_dqo()
124 dma_free_coherent(hdev, bytes, tx->dqo.compl_ring, in gve_tx_free_ring_dqo()
125 tx->complq_bus_dqo); in gve_tx_free_ring_dqo()
126 tx->dqo.compl_ring = NULL; in gve_tx_free_ring_dqo()
129 if (tx->dqo.tx_ring) { in gve_tx_free_ring_dqo()
130 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); in gve_tx_free_ring_dqo()
131 dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus); in gve_tx_free_ring_dqo()
132 tx->dqo.tx_ring = NULL; in gve_tx_free_ring_dqo()
135 kvfree(tx->dqo.pending_packets); in gve_tx_free_ring_dqo()
136 tx->dqo.pending_packets = NULL; in gve_tx_free_ring_dqo()
138 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); in gve_tx_free_ring_dqo()
143 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_alloc_ring_dqo() local
149 memset(tx, 0, sizeof(*tx)); in gve_tx_alloc_ring_dqo()
150 tx->q_num = idx; in gve_tx_alloc_ring_dqo()
151 tx->dev = &priv->pdev->dev; in gve_tx_alloc_ring_dqo()
152 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx); in gve_tx_alloc_ring_dqo()
153 atomic_set_release(&tx->dqo_compl.hw_tx_head, 0); in gve_tx_alloc_ring_dqo()
156 tx->mask = priv->tx_desc_cnt - 1; in gve_tx_alloc_ring_dqo()
157 tx->dqo.complq_mask = priv->options_dqo_rda.tx_comp_ring_entries - 1; in gve_tx_alloc_ring_dqo()
165 num_pending_packets = tx->dqo.complq_mask + 1; in gve_tx_alloc_ring_dqo()
171 (tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL; in gve_tx_alloc_ring_dqo()
178 tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX); in gve_tx_alloc_ring_dqo()
179 tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets, in gve_tx_alloc_ring_dqo()
180 sizeof(tx->dqo.pending_packets[0]), in gve_tx_alloc_ring_dqo()
182 if (!tx->dqo.pending_packets) in gve_tx_alloc_ring_dqo()
186 for (i = 0; i < tx->dqo.num_pending_packets - 1; i++) in gve_tx_alloc_ring_dqo()
187 tx->dqo.pending_packets[i].next = i + 1; in gve_tx_alloc_ring_dqo()
189 tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1; in gve_tx_alloc_ring_dqo()
190 atomic_set_release(&tx->dqo_compl.free_pending_packets, -1); in gve_tx_alloc_ring_dqo()
191 tx->dqo_compl.miss_completions.head = -1; in gve_tx_alloc_ring_dqo()
192 tx->dqo_compl.miss_completions.tail = -1; in gve_tx_alloc_ring_dqo()
193 tx->dqo_compl.timed_out_completions.head = -1; in gve_tx_alloc_ring_dqo()
194 tx->dqo_compl.timed_out_completions.tail = -1; in gve_tx_alloc_ring_dqo()
196 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); in gve_tx_alloc_ring_dqo()
197 tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL); in gve_tx_alloc_ring_dqo()
198 if (!tx->dqo.tx_ring) in gve_tx_alloc_ring_dqo()
201 bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1); in gve_tx_alloc_ring_dqo()
202 tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes, in gve_tx_alloc_ring_dqo()
203 &tx->complq_bus_dqo, in gve_tx_alloc_ring_dqo()
205 if (!tx->dqo.compl_ring) in gve_tx_alloc_ring_dqo()
208 tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources), in gve_tx_alloc_ring_dqo()
209 &tx->q_resources_bus, GFP_KERNEL); in gve_tx_alloc_ring_dqo()
210 if (!tx->q_resources) in gve_tx_alloc_ring_dqo()
231 "Failed to alloc tx ring=%d: err=%d\n", in gve_tx_alloc_rings_dqo()
251 struct gve_tx_ring *tx = &priv->tx[i]; in gve_tx_free_rings_dqo() local
253 gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL); in gve_tx_free_rings_dqo()
254 netdev_tx_reset_queue(tx->netdev_txq); in gve_tx_free_rings_dqo()
255 gve_tx_clean_pending_packets(tx); in gve_tx_free_rings_dqo()
262 static u32 num_avail_tx_slots(const struct gve_tx_ring *tx) in num_avail_tx_slots() argument
264 u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask; in num_avail_tx_slots()
266 return tx->mask - num_used; in num_avail_tx_slots()
272 static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx, int count) in gve_maybe_stop_tx_dqo() argument
274 if (likely(gve_has_pending_packet(tx) && in gve_maybe_stop_tx_dqo()
275 num_avail_tx_slots(tx) >= count)) in gve_maybe_stop_tx_dqo()
278 /* Update cached TX head pointer */ in gve_maybe_stop_tx_dqo()
279 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); in gve_maybe_stop_tx_dqo()
281 if (likely(gve_has_pending_packet(tx) && in gve_maybe_stop_tx_dqo()
282 num_avail_tx_slots(tx) >= count)) in gve_maybe_stop_tx_dqo()
286 tx->stop_queue++; in gve_maybe_stop_tx_dqo()
287 netif_tx_stop_queue(tx->netdev_txq); in gve_maybe_stop_tx_dqo()
295 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); in gve_maybe_stop_tx_dqo()
297 if (likely(!gve_has_pending_packet(tx) || in gve_maybe_stop_tx_dqo()
298 num_avail_tx_slots(tx) < count)) in gve_maybe_stop_tx_dqo()
301 netif_tx_start_queue(tx->netdev_txq); in gve_maybe_stop_tx_dqo()
302 tx->wake_queue++; in gve_maybe_stop_tx_dqo()
323 static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, in gve_tx_fill_pkt_desc_dqo() argument
331 &tx->dqo.tx_ring[*desc_idx].pkt; in gve_tx_fill_pkt_desc_dqo()
346 *desc_idx = (*desc_idx + 1) & tx->mask; in gve_tx_fill_pkt_desc_dqo()
449 * gve_has_pending_packet(tx) returns true.
451 static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, in gve_tx_add_skb_no_copy_dqo() argument
456 u32 desc_idx = tx->dqo_tx.tail; in gve_tx_add_skb_no_copy_dqo()
463 pkt = gve_alloc_pending_packet(tx); in gve_tx_add_skb_no_copy_dqo()
466 completion_tag = pkt - tx->dqo.pending_packets; in gve_tx_add_skb_no_copy_dqo()
475 gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx, in gve_tx_add_skb_no_copy_dqo()
477 desc_idx = (desc_idx + 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
480 gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx, in gve_tx_add_skb_no_copy_dqo()
482 desc_idx = (desc_idx + 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
497 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy_dqo()
498 if (unlikely(dma_mapping_error(tx->dev, addr))) in gve_tx_add_skb_no_copy_dqo()
505 gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, in gve_tx_add_skb_no_copy_dqo()
516 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy_dqo()
517 if (unlikely(dma_mapping_error(tx->dev, addr))) in gve_tx_add_skb_no_copy_dqo()
524 gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, in gve_tx_add_skb_no_copy_dqo()
529 tx->dqo_tx.tail = desc_idx; in gve_tx_add_skb_no_copy_dqo()
535 u32 last_desc_idx = (desc_idx - 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
537 (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
541 tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; in gve_tx_add_skb_no_copy_dqo()
542 tx->dqo_tx.last_re_idx = last_desc_idx; in gve_tx_add_skb_no_copy_dqo()
551 dma_unmap_single(tx->dev, in gve_tx_add_skb_no_copy_dqo()
556 dma_unmap_page(tx->dev, in gve_tx_add_skb_no_copy_dqo()
565 gve_free_pending_packet(tx, pkt); in gve_tx_add_skb_no_copy_dqo()
632 static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_try_tx_skb() argument
663 if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs + in gve_try_tx_skb()
668 if (unlikely(gve_tx_add_skb_no_copy_dqo(tx, skb) < 0)) in gve_try_tx_skb()
671 netdev_tx_sent_queue(tx->netdev_txq, skb->len); in gve_try_tx_skb()
676 tx->dropped_pkt++; in gve_try_tx_skb()
685 struct gve_tx_ring *tx; in gve_tx_dqo() local
687 tx = &priv->tx[skb_get_queue_mapping(skb)]; in gve_tx_dqo()
688 if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) { in gve_tx_dqo()
689 /* We need to ring the txq doorbell -- we have stopped the Tx in gve_tx_dqo()
693 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); in gve_tx_dqo()
697 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more()) in gve_tx_dqo()
700 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); in gve_tx_dqo()
704 static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list, in add_to_list() argument
709 index = pending_packet - tx->dqo.pending_packets; in add_to_list()
715 tx->dqo.pending_packets[old_tail].next = index; in add_to_list()
721 static void remove_from_list(struct gve_tx_ring *tx, in remove_from_list() argument
734 tx->dqo.pending_packets[prev_index].next = next_index; in remove_from_list()
740 tx->dqo.pending_packets[next_index].prev = prev_index; in remove_from_list()
766 struct gve_tx_ring *tx, bool is_napi, in gve_handle_packet_completion() argument
772 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { in gve_handle_packet_completion()
773 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", in gve_handle_packet_completion()
778 pending_packet = &tx->dqo.pending_packets[compl_tag]; in gve_handle_packet_completion()
788 remove_from_list(tx, in gve_handle_packet_completion()
789 &tx->dqo_compl.timed_out_completions, in gve_handle_packet_completion()
791 gve_free_pending_packet(tx, pending_packet); in gve_handle_packet_completion()
805 remove_from_list(tx, &tx->dqo_compl.miss_completions, in gve_handle_packet_completion()
816 gve_unmap_packet(tx->dev, pending_packet); in gve_handle_packet_completion()
822 gve_free_pending_packet(tx, pending_packet); in gve_handle_packet_completion()
826 struct gve_tx_ring *tx, u16 compl_tag, in gve_handle_miss_completion() argument
831 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { in gve_handle_miss_completion()
832 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", in gve_handle_miss_completion()
837 pending_packet = &tx->dqo.pending_packets[compl_tag]; in gve_handle_miss_completion()
852 add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet); in gve_handle_miss_completion()
859 struct gve_tx_ring *tx) in remove_miss_completions() argument
864 next_index = tx->dqo_compl.miss_completions.head; in remove_miss_completions()
866 pending_packet = &tx->dqo.pending_packets[next_index]; in remove_miss_completions()
872 remove_from_list(tx, &tx->dqo_compl.miss_completions, in remove_miss_completions()
879 gve_unmap_packet(tx->dev, pending_packet); in remove_miss_completions()
883 tx->dropped_pkt++; in remove_miss_completions()
886 (int)(pending_packet - tx->dqo.pending_packets)); in remove_miss_completions()
896 add_to_list(tx, &tx->dqo_compl.timed_out_completions, in remove_miss_completions()
902 struct gve_tx_ring *tx) in remove_timed_out_completions() argument
907 next_index = tx->dqo_compl.timed_out_completions.head; in remove_timed_out_completions()
909 pending_packet = &tx->dqo.pending_packets[next_index]; in remove_timed_out_completions()
915 remove_from_list(tx, &tx->dqo_compl.timed_out_completions, in remove_timed_out_completions()
917 gve_free_pending_packet(tx, pending_packet); in remove_timed_out_completions()
921 int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_clean_tx_done_dqo() argument
935 &tx->dqo.compl_ring[tx->dqo_compl.head]; in gve_clean_tx_done_dqo()
938 if (compl_desc->generation == tx->dqo_compl.cur_gen_bit) in gve_clean_tx_done_dqo()
942 prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) & in gve_clean_tx_done_dqo()
943 tx->dqo.complq_mask]); in gve_clean_tx_done_dqo()
953 atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head); in gve_clean_tx_done_dqo()
957 gve_handle_packet_completion(priv, tx, !!napi, in gve_clean_tx_done_dqo()
965 gve_handle_miss_completion(priv, tx, compl_tag, in gve_clean_tx_done_dqo()
971 gve_handle_packet_completion(priv, tx, !!napi, in gve_clean_tx_done_dqo()
978 tx->dqo_compl.head = in gve_clean_tx_done_dqo()
979 (tx->dqo_compl.head + 1) & tx->dqo.complq_mask; in gve_clean_tx_done_dqo()
981 tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0; in gve_clean_tx_done_dqo()
985 netdev_tx_completed_queue(tx->netdev_txq, in gve_clean_tx_done_dqo()
989 remove_miss_completions(priv, tx); in gve_clean_tx_done_dqo()
990 remove_timed_out_completions(priv, tx); in gve_clean_tx_done_dqo()
992 u64_stats_update_begin(&tx->statss); in gve_clean_tx_done_dqo()
993 tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes; in gve_clean_tx_done_dqo()
994 tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts; in gve_clean_tx_done_dqo()
995 u64_stats_update_end(&tx->statss); in gve_clean_tx_done_dqo()
1002 struct gve_tx_ring *tx = block->tx; in gve_tx_poll_dqo() local
1006 int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx, in gve_tx_poll_dqo()
1012 if (netif_tx_queue_stopped(tx->netdev_txq) && in gve_tx_poll_dqo()
1014 tx->wake_queue++; in gve_tx_poll_dqo()
1015 netif_tx_wake_queue(tx->netdev_txq); in gve_tx_poll_dqo()
1020 compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head]; in gve_tx_poll_dqo()
1021 return compl_desc->generation != tx->dqo_compl.cur_gen_bit; in gve_tx_poll_dqo()