Lines Matching full:tx
16 static bool gve_has_pending_packet(struct gve_tx_ring *tx) in gve_has_pending_packet() argument
18 /* Check TX path's list. */ in gve_has_pending_packet()
19 if (tx->dqo_tx.free_pending_packets != -1) in gve_has_pending_packet()
23 if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1) in gve_has_pending_packet()
30 gve_alloc_pending_packet(struct gve_tx_ring *tx) in gve_alloc_pending_packet() argument
35 index = tx->dqo_tx.free_pending_packets; in gve_alloc_pending_packet()
41 tx->dqo_tx.free_pending_packets = in gve_alloc_pending_packet()
42 atomic_xchg(&tx->dqo_compl.free_pending_packets, -1); in gve_alloc_pending_packet()
43 index = tx->dqo_tx.free_pending_packets; in gve_alloc_pending_packet()
49 pending_packet = &tx->dqo.pending_packets[index]; in gve_alloc_pending_packet()
52 tx->dqo_tx.free_pending_packets = pending_packet->next; in gve_alloc_pending_packet()
59 gve_free_pending_packet(struct gve_tx_ring *tx, in gve_free_pending_packet() argument
62 s16 index = pending_packet - tx->dqo.pending_packets; in gve_free_pending_packet()
66 s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets); in gve_free_pending_packet()
69 if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets, in gve_free_pending_packet()
76 /* gve_tx_free_desc - Cleans up all pending tx requests and buffers.
78 static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) in gve_tx_clean_pending_packets() argument
82 for (i = 0; i < tx->dqo.num_pending_packets; i++) { in gve_tx_clean_pending_packets()
84 &tx->dqo.pending_packets[i]; in gve_tx_clean_pending_packets()
91 dma_unmap_single(tx->dev, in gve_tx_clean_pending_packets()
96 dma_unmap_page(tx->dev, in gve_tx_clean_pending_packets()
111 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_free_ring_dqo() local
117 if (tx->q_resources) { in gve_tx_free_ring_dqo()
118 dma_free_coherent(hdev, sizeof(*tx->q_resources), in gve_tx_free_ring_dqo()
119 tx->q_resources, tx->q_resources_bus); in gve_tx_free_ring_dqo()
120 tx->q_resources = NULL; in gve_tx_free_ring_dqo()
123 if (tx->dqo.compl_ring) { in gve_tx_free_ring_dqo()
124 bytes = sizeof(tx->dqo.compl_ring[0]) * in gve_tx_free_ring_dqo()
125 (tx->dqo.complq_mask + 1); in gve_tx_free_ring_dqo()
126 dma_free_coherent(hdev, bytes, tx->dqo.compl_ring, in gve_tx_free_ring_dqo()
127 tx->complq_bus_dqo); in gve_tx_free_ring_dqo()
128 tx->dqo.compl_ring = NULL; in gve_tx_free_ring_dqo()
131 if (tx->dqo.tx_ring) { in gve_tx_free_ring_dqo()
132 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); in gve_tx_free_ring_dqo()
133 dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus); in gve_tx_free_ring_dqo()
134 tx->dqo.tx_ring = NULL; in gve_tx_free_ring_dqo()
137 kvfree(tx->dqo.pending_packets); in gve_tx_free_ring_dqo()
138 tx->dqo.pending_packets = NULL; in gve_tx_free_ring_dqo()
140 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); in gve_tx_free_ring_dqo()
145 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_alloc_ring_dqo() local
151 memset(tx, 0, sizeof(*tx)); in gve_tx_alloc_ring_dqo()
152 tx->q_num = idx; in gve_tx_alloc_ring_dqo()
153 tx->dev = &priv->pdev->dev; in gve_tx_alloc_ring_dqo()
154 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx); in gve_tx_alloc_ring_dqo()
155 atomic_set_release(&tx->dqo_compl.hw_tx_head, 0); in gve_tx_alloc_ring_dqo()
158 tx->mask = priv->tx_desc_cnt - 1; in gve_tx_alloc_ring_dqo()
159 tx->dqo.complq_mask = priv->options_dqo_rda.tx_comp_ring_entries - 1; in gve_tx_alloc_ring_dqo()
167 num_pending_packets = tx->dqo.complq_mask + 1; in gve_tx_alloc_ring_dqo()
173 (tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL; in gve_tx_alloc_ring_dqo()
180 tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX); in gve_tx_alloc_ring_dqo()
181 tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets, in gve_tx_alloc_ring_dqo()
182 sizeof(tx->dqo.pending_packets[0]), in gve_tx_alloc_ring_dqo()
184 if (!tx->dqo.pending_packets) in gve_tx_alloc_ring_dqo()
188 for (i = 0; i < tx->dqo.num_pending_packets - 1; i++) in gve_tx_alloc_ring_dqo()
189 tx->dqo.pending_packets[i].next = i + 1; in gve_tx_alloc_ring_dqo()
191 tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1; in gve_tx_alloc_ring_dqo()
192 atomic_set_release(&tx->dqo_compl.free_pending_packets, -1); in gve_tx_alloc_ring_dqo()
193 tx->dqo_compl.miss_completions.head = -1; in gve_tx_alloc_ring_dqo()
194 tx->dqo_compl.miss_completions.tail = -1; in gve_tx_alloc_ring_dqo()
195 tx->dqo_compl.timed_out_completions.head = -1; in gve_tx_alloc_ring_dqo()
196 tx->dqo_compl.timed_out_completions.tail = -1; in gve_tx_alloc_ring_dqo()
198 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); in gve_tx_alloc_ring_dqo()
199 tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL); in gve_tx_alloc_ring_dqo()
200 if (!tx->dqo.tx_ring) in gve_tx_alloc_ring_dqo()
203 bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1); in gve_tx_alloc_ring_dqo()
204 tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes, in gve_tx_alloc_ring_dqo()
205 &tx->complq_bus_dqo, in gve_tx_alloc_ring_dqo()
207 if (!tx->dqo.compl_ring) in gve_tx_alloc_ring_dqo()
210 tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources), in gve_tx_alloc_ring_dqo()
211 &tx->q_resources_bus, GFP_KERNEL); in gve_tx_alloc_ring_dqo()
212 if (!tx->q_resources) in gve_tx_alloc_ring_dqo()
233 "Failed to alloc tx ring=%d: err=%d\n", in gve_tx_alloc_rings_dqo()
253 struct gve_tx_ring *tx = &priv->tx[i]; in gve_tx_free_rings_dqo() local
255 gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL); in gve_tx_free_rings_dqo()
256 netdev_tx_reset_queue(tx->netdev_txq); in gve_tx_free_rings_dqo()
257 gve_tx_clean_pending_packets(tx); in gve_tx_free_rings_dqo()
264 static u32 num_avail_tx_slots(const struct gve_tx_ring *tx) in num_avail_tx_slots() argument
266 u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask; in num_avail_tx_slots()
268 return tx->mask - num_used; in num_avail_tx_slots()
274 static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx, int count) in gve_maybe_stop_tx_dqo() argument
276 if (likely(gve_has_pending_packet(tx) && in gve_maybe_stop_tx_dqo()
277 num_avail_tx_slots(tx) >= count)) in gve_maybe_stop_tx_dqo()
280 /* Update cached TX head pointer */ in gve_maybe_stop_tx_dqo()
281 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); in gve_maybe_stop_tx_dqo()
283 if (likely(gve_has_pending_packet(tx) && in gve_maybe_stop_tx_dqo()
284 num_avail_tx_slots(tx) >= count)) in gve_maybe_stop_tx_dqo()
288 tx->stop_queue++; in gve_maybe_stop_tx_dqo()
289 netif_tx_stop_queue(tx->netdev_txq); in gve_maybe_stop_tx_dqo()
297 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); in gve_maybe_stop_tx_dqo()
299 if (likely(!gve_has_pending_packet(tx) || in gve_maybe_stop_tx_dqo()
300 num_avail_tx_slots(tx) < count)) in gve_maybe_stop_tx_dqo()
303 netif_tx_start_queue(tx->netdev_txq); in gve_maybe_stop_tx_dqo()
304 tx->wake_queue++; in gve_maybe_stop_tx_dqo()
325 static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, in gve_tx_fill_pkt_desc_dqo() argument
333 &tx->dqo.tx_ring[*desc_idx].pkt; in gve_tx_fill_pkt_desc_dqo()
348 *desc_idx = (*desc_idx + 1) & tx->mask; in gve_tx_fill_pkt_desc_dqo()
451 * gve_has_pending_packet(tx) returns true.
453 static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, in gve_tx_add_skb_no_copy_dqo() argument
458 u32 desc_idx = tx->dqo_tx.tail; in gve_tx_add_skb_no_copy_dqo()
465 pending_packet = gve_alloc_pending_packet(tx); in gve_tx_add_skb_no_copy_dqo()
468 completion_tag = pending_packet - tx->dqo.pending_packets; in gve_tx_add_skb_no_copy_dqo()
477 gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx, in gve_tx_add_skb_no_copy_dqo()
479 desc_idx = (desc_idx + 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
482 gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx, in gve_tx_add_skb_no_copy_dqo()
484 desc_idx = (desc_idx + 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
501 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy_dqo()
502 if (unlikely(dma_mapping_error(tx->dev, addr))) in gve_tx_add_skb_no_copy_dqo()
509 gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, in gve_tx_add_skb_no_copy_dqo()
522 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy_dqo()
523 if (unlikely(dma_mapping_error(tx->dev, addr))) in gve_tx_add_skb_no_copy_dqo()
530 gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, in gve_tx_add_skb_no_copy_dqo()
535 tx->dqo_tx.tail = desc_idx; in gve_tx_add_skb_no_copy_dqo()
541 u32 last_desc_idx = (desc_idx - 1) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
543 (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; in gve_tx_add_skb_no_copy_dqo()
547 tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; in gve_tx_add_skb_no_copy_dqo()
548 tx->dqo_tx.last_re_idx = last_desc_idx; in gve_tx_add_skb_no_copy_dqo()
559 dma_unmap_single(tx->dev, dma_unmap_addr(buf, dma), in gve_tx_add_skb_no_copy_dqo()
563 dma_unmap_page(tx->dev, dma_unmap_addr(buf, dma), in gve_tx_add_skb_no_copy_dqo()
570 gve_free_pending_packet(tx, pending_packet); in gve_tx_add_skb_no_copy_dqo()
637 static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_try_tx_skb() argument
668 if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs + in gve_try_tx_skb()
673 if (unlikely(gve_tx_add_skb_no_copy_dqo(tx, skb) < 0)) in gve_try_tx_skb()
676 netdev_tx_sent_queue(tx->netdev_txq, skb->len); in gve_try_tx_skb()
681 tx->dropped_pkt++; in gve_try_tx_skb()
690 struct gve_tx_ring *tx; in gve_tx_dqo() local
692 tx = &priv->tx[skb_get_queue_mapping(skb)]; in gve_tx_dqo()
693 if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) { in gve_tx_dqo()
694 /* We need to ring the txq doorbell -- we have stopped the Tx in gve_tx_dqo()
698 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); in gve_tx_dqo()
702 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more()) in gve_tx_dqo()
705 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); in gve_tx_dqo()
709 static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list, in add_to_list() argument
714 index = pending_packet - tx->dqo.pending_packets; in add_to_list()
720 tx->dqo.pending_packets[old_tail].next = index; in add_to_list()
726 static void remove_from_list(struct gve_tx_ring *tx, in remove_from_list() argument
739 tx->dqo.pending_packets[prev_index].next = next_index; in remove_from_list()
745 tx->dqo.pending_packets[next_index].prev = prev_index; in remove_from_list()
774 struct gve_tx_ring *tx, bool is_napi, in gve_handle_packet_completion() argument
780 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { in gve_handle_packet_completion()
781 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", in gve_handle_packet_completion()
786 pending_packet = &tx->dqo.pending_packets[compl_tag]; in gve_handle_packet_completion()
796 remove_from_list(tx, in gve_handle_packet_completion()
797 &tx->dqo_compl.timed_out_completions, in gve_handle_packet_completion()
799 gve_free_pending_packet(tx, pending_packet); in gve_handle_packet_completion()
813 remove_from_list(tx, &tx->dqo_compl.miss_completions, in gve_handle_packet_completion()
824 gve_unmap_packet(tx->dev, pending_packet); in gve_handle_packet_completion()
830 gve_free_pending_packet(tx, pending_packet); in gve_handle_packet_completion()
834 struct gve_tx_ring *tx, u16 compl_tag, in gve_handle_miss_completion() argument
839 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { in gve_handle_miss_completion()
840 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", in gve_handle_miss_completion()
845 pending_packet = &tx->dqo.pending_packets[compl_tag]; in gve_handle_miss_completion()
860 add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet); in gve_handle_miss_completion()
867 struct gve_tx_ring *tx) in remove_miss_completions() argument
872 next_index = tx->dqo_compl.miss_completions.head; in remove_miss_completions()
874 pending_packet = &tx->dqo.pending_packets[next_index]; in remove_miss_completions()
880 remove_from_list(tx, &tx->dqo_compl.miss_completions, in remove_miss_completions()
887 gve_unmap_packet(tx->dev, pending_packet); in remove_miss_completions()
891 tx->dropped_pkt++; in remove_miss_completions()
894 (int)(pending_packet - tx->dqo.pending_packets)); in remove_miss_completions()
904 add_to_list(tx, &tx->dqo_compl.timed_out_completions, in remove_miss_completions()
910 struct gve_tx_ring *tx) in remove_timed_out_completions() argument
915 next_index = tx->dqo_compl.timed_out_completions.head; in remove_timed_out_completions()
917 pending_packet = &tx->dqo.pending_packets[next_index]; in remove_timed_out_completions()
923 remove_from_list(tx, &tx->dqo_compl.timed_out_completions, in remove_timed_out_completions()
925 gve_free_pending_packet(tx, pending_packet); in remove_timed_out_completions()
929 int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_clean_tx_done_dqo() argument
943 &tx->dqo.compl_ring[tx->dqo_compl.head]; in gve_clean_tx_done_dqo()
946 if (compl_desc->generation == tx->dqo_compl.cur_gen_bit) in gve_clean_tx_done_dqo()
950 prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) & in gve_clean_tx_done_dqo()
951 tx->dqo.complq_mask]); in gve_clean_tx_done_dqo()
961 atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head); in gve_clean_tx_done_dqo()
965 gve_handle_packet_completion(priv, tx, !!napi, in gve_clean_tx_done_dqo()
973 gve_handle_miss_completion(priv, tx, compl_tag, in gve_clean_tx_done_dqo()
979 gve_handle_packet_completion(priv, tx, !!napi, in gve_clean_tx_done_dqo()
986 tx->dqo_compl.head = in gve_clean_tx_done_dqo()
987 (tx->dqo_compl.head + 1) & tx->dqo.complq_mask; in gve_clean_tx_done_dqo()
989 tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0; in gve_clean_tx_done_dqo()
993 netdev_tx_completed_queue(tx->netdev_txq, in gve_clean_tx_done_dqo()
997 remove_miss_completions(priv, tx); in gve_clean_tx_done_dqo()
998 remove_timed_out_completions(priv, tx); in gve_clean_tx_done_dqo()
1000 u64_stats_update_begin(&tx->statss); in gve_clean_tx_done_dqo()
1001 tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes; in gve_clean_tx_done_dqo()
1002 tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts; in gve_clean_tx_done_dqo()
1003 u64_stats_update_end(&tx->statss); in gve_clean_tx_done_dqo()
1010 struct gve_tx_ring *tx = block->tx; in gve_tx_poll_dqo() local
1014 int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx, in gve_tx_poll_dqo()
1020 if (netif_tx_queue_stopped(tx->netdev_txq) && in gve_tx_poll_dqo()
1022 tx->wake_queue++; in gve_tx_poll_dqo()
1023 netif_tx_wake_queue(tx->netdev_txq); in gve_tx_poll_dqo()
1028 compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head]; in gve_tx_poll_dqo()
1029 return compl_desc->generation != tx->dqo_compl.cur_gen_bit; in gve_tx_poll_dqo()