1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2021 Google, Inc.
5 */
6
7 #include "gve.h"
8 #include "gve_adminq.h"
9 #include "gve_utils.h"
10 #include <linux/ip.h>
11 #include <linux/tcp.h>
12 #include <linux/vmalloc.h>
13 #include <linux/skbuff.h>
14 #include <net/xdp_sock_drv.h>
15
gve_tx_put_doorbell(struct gve_priv * priv,struct gve_queue_resources * q_resources,u32 val)16 static inline void gve_tx_put_doorbell(struct gve_priv *priv,
17 struct gve_queue_resources *q_resources,
18 u32 val)
19 {
20 iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]);
21 }
22
gve_xdp_tx_flush(struct gve_priv * priv,u32 xdp_qid)23 void gve_xdp_tx_flush(struct gve_priv *priv, u32 xdp_qid)
24 {
25 u32 tx_qid = gve_xdp_tx_queue_id(priv, xdp_qid);
26 struct gve_tx_ring *tx = &priv->tx[tx_qid];
27
28 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
29 }
30
31 /* gvnic can only transmit from a Registered Segment.
32 * We copy skb payloads into the registered segment before writing Tx
33 * descriptors and ringing the Tx doorbell.
34 *
35 * gve_tx_fifo_* manages the Registered Segment as a FIFO - clients must
36 * free allocations in the order they were allocated.
37 */
38
gve_tx_fifo_init(struct gve_priv * priv,struct gve_tx_fifo * fifo)39 static int gve_tx_fifo_init(struct gve_priv *priv, struct gve_tx_fifo *fifo)
40 {
41 fifo->base = vmap(fifo->qpl->pages, fifo->qpl->num_entries, VM_MAP,
42 PAGE_KERNEL);
43 if (unlikely(!fifo->base)) {
44 netif_err(priv, drv, priv->dev, "Failed to vmap fifo, qpl_id = %d\n",
45 fifo->qpl->id);
46 return -ENOMEM;
47 }
48
49 fifo->size = fifo->qpl->num_entries * PAGE_SIZE;
50 atomic_set(&fifo->available, fifo->size);
51 fifo->head = 0;
52 return 0;
53 }
54
gve_tx_fifo_release(struct gve_priv * priv,struct gve_tx_fifo * fifo)55 static void gve_tx_fifo_release(struct gve_priv *priv, struct gve_tx_fifo *fifo)
56 {
57 WARN(atomic_read(&fifo->available) != fifo->size,
58 "Releasing non-empty fifo");
59
60 vunmap(fifo->base);
61 }
62
gve_tx_fifo_pad_alloc_one_frag(struct gve_tx_fifo * fifo,size_t bytes)63 static int gve_tx_fifo_pad_alloc_one_frag(struct gve_tx_fifo *fifo,
64 size_t bytes)
65 {
66 return (fifo->head + bytes < fifo->size) ? 0 : fifo->size - fifo->head;
67 }
68
gve_tx_fifo_can_alloc(struct gve_tx_fifo * fifo,size_t bytes)69 static bool gve_tx_fifo_can_alloc(struct gve_tx_fifo *fifo, size_t bytes)
70 {
71 return (atomic_read(&fifo->available) <= bytes) ? false : true;
72 }
73
74 /* gve_tx_alloc_fifo - Allocate fragment(s) from Tx FIFO
75 * @fifo: FIFO to allocate from
76 * @bytes: Allocation size
77 * @iov: Scatter-gather elements to fill with allocation fragment base/len
78 *
79 * Returns number of valid elements in iov[] or negative on error.
80 *
81 * Allocations from a given FIFO must be externally synchronized but concurrent
82 * allocation and frees are allowed.
83 */
gve_tx_alloc_fifo(struct gve_tx_fifo * fifo,size_t bytes,struct gve_tx_iovec iov[2])84 static int gve_tx_alloc_fifo(struct gve_tx_fifo *fifo, size_t bytes,
85 struct gve_tx_iovec iov[2])
86 {
87 size_t overflow, padding;
88 u32 aligned_head;
89 int nfrags = 0;
90
91 if (!bytes)
92 return 0;
93
94 /* This check happens before we know how much padding is needed to
95 * align to a cacheline boundary for the payload, but that is fine,
96 * because the FIFO head always start aligned, and the FIFO's boundaries
97 * are aligned, so if there is space for the data, there is space for
98 * the padding to the next alignment.
99 */
100 WARN(!gve_tx_fifo_can_alloc(fifo, bytes),
101 "Reached %s when there's not enough space in the fifo", __func__);
102
103 nfrags++;
104
105 iov[0].iov_offset = fifo->head;
106 iov[0].iov_len = bytes;
107 fifo->head += bytes;
108
109 if (fifo->head > fifo->size) {
110 /* If the allocation did not fit in the tail fragment of the
111 * FIFO, also use the head fragment.
112 */
113 nfrags++;
114 overflow = fifo->head - fifo->size;
115 iov[0].iov_len -= overflow;
116 iov[1].iov_offset = 0; /* Start of fifo*/
117 iov[1].iov_len = overflow;
118
119 fifo->head = overflow;
120 }
121
122 /* Re-align to a cacheline boundary */
123 aligned_head = L1_CACHE_ALIGN(fifo->head);
124 padding = aligned_head - fifo->head;
125 iov[nfrags - 1].iov_padding = padding;
126 atomic_sub(bytes + padding, &fifo->available);
127 fifo->head = aligned_head;
128
129 if (fifo->head == fifo->size)
130 fifo->head = 0;
131
132 return nfrags;
133 }
134
135 /* gve_tx_free_fifo - Return space to Tx FIFO
136 * @fifo: FIFO to return fragments to
137 * @bytes: Bytes to free
138 */
gve_tx_free_fifo(struct gve_tx_fifo * fifo,size_t bytes)139 static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes)
140 {
141 atomic_add(bytes, &fifo->available);
142 }
143
gve_tx_clear_buffer_state(struct gve_tx_buffer_state * info)144 static size_t gve_tx_clear_buffer_state(struct gve_tx_buffer_state *info)
145 {
146 size_t space_freed = 0;
147 int i;
148
149 for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
150 space_freed += info->iov[i].iov_len + info->iov[i].iov_padding;
151 info->iov[i].iov_len = 0;
152 info->iov[i].iov_padding = 0;
153 }
154 return space_freed;
155 }
156
gve_clean_xdp_done(struct gve_priv * priv,struct gve_tx_ring * tx,u32 to_do)157 static int gve_clean_xdp_done(struct gve_priv *priv, struct gve_tx_ring *tx,
158 u32 to_do)
159 {
160 struct gve_tx_buffer_state *info;
161 u32 clean_end = tx->done + to_do;
162 u64 pkts = 0, bytes = 0;
163 size_t space_freed = 0;
164 u32 xsk_complete = 0;
165 u32 idx;
166
167 for (; tx->done < clean_end; tx->done++) {
168 idx = tx->done & tx->mask;
169 info = &tx->info[idx];
170
171 if (unlikely(!info->xdp.size))
172 continue;
173
174 bytes += info->xdp.size;
175 pkts++;
176 xsk_complete += info->xdp.is_xsk;
177
178 info->xdp.size = 0;
179 if (info->xdp_frame) {
180 xdp_return_frame(info->xdp_frame);
181 info->xdp_frame = NULL;
182 }
183 space_freed += gve_tx_clear_buffer_state(info);
184 }
185
186 gve_tx_free_fifo(&tx->tx_fifo, space_freed);
187 if (xsk_complete > 0 && tx->xsk_pool)
188 xsk_tx_completed(tx->xsk_pool, xsk_complete);
189 u64_stats_update_begin(&tx->statss);
190 tx->bytes_done += bytes;
191 tx->pkt_done += pkts;
192 u64_stats_update_end(&tx->statss);
193 return pkts;
194 }
195
196 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
197 u32 to_do, bool try_to_wake);
198
gve_tx_free_ring(struct gve_priv * priv,int idx)199 static void gve_tx_free_ring(struct gve_priv *priv, int idx)
200 {
201 struct gve_tx_ring *tx = &priv->tx[idx];
202 struct device *hdev = &priv->pdev->dev;
203 size_t bytes;
204 u32 slots;
205
206 gve_tx_remove_from_block(priv, idx);
207 slots = tx->mask + 1;
208 if (tx->q_num < priv->tx_cfg.num_queues) {
209 gve_clean_tx_done(priv, tx, priv->tx_desc_cnt, false);
210 netdev_tx_reset_queue(tx->netdev_txq);
211 } else {
212 gve_clean_xdp_done(priv, tx, priv->tx_desc_cnt);
213 }
214
215 dma_free_coherent(hdev, sizeof(*tx->q_resources),
216 tx->q_resources, tx->q_resources_bus);
217 tx->q_resources = NULL;
218
219 if (!tx->raw_addressing) {
220 gve_tx_fifo_release(priv, &tx->tx_fifo);
221 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
222 tx->tx_fifo.qpl = NULL;
223 }
224
225 bytes = sizeof(*tx->desc) * slots;
226 dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
227 tx->desc = NULL;
228
229 vfree(tx->info);
230 tx->info = NULL;
231
232 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
233 }
234
gve_tx_alloc_ring(struct gve_priv * priv,int idx)235 static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
236 {
237 struct gve_tx_ring *tx = &priv->tx[idx];
238 struct device *hdev = &priv->pdev->dev;
239 u32 slots = priv->tx_desc_cnt;
240 size_t bytes;
241
242 /* Make sure everything is zeroed to start */
243 memset(tx, 0, sizeof(*tx));
244 spin_lock_init(&tx->clean_lock);
245 spin_lock_init(&tx->xdp_lock);
246 tx->q_num = idx;
247
248 tx->mask = slots - 1;
249
250 /* alloc metadata */
251 tx->info = vcalloc(slots, sizeof(*tx->info));
252 if (!tx->info)
253 return -ENOMEM;
254
255 /* alloc tx queue */
256 bytes = sizeof(*tx->desc) * slots;
257 tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
258 if (!tx->desc)
259 goto abort_with_info;
260
261 tx->raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
262 tx->dev = &priv->pdev->dev;
263 if (!tx->raw_addressing) {
264 tx->tx_fifo.qpl = gve_assign_tx_qpl(priv, idx);
265 if (!tx->tx_fifo.qpl)
266 goto abort_with_desc;
267 /* map Tx FIFO */
268 if (gve_tx_fifo_init(priv, &tx->tx_fifo))
269 goto abort_with_qpl;
270 }
271
272 tx->q_resources =
273 dma_alloc_coherent(hdev,
274 sizeof(*tx->q_resources),
275 &tx->q_resources_bus,
276 GFP_KERNEL);
277 if (!tx->q_resources)
278 goto abort_with_fifo;
279
280 netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx,
281 (unsigned long)tx->bus);
282 if (idx < priv->tx_cfg.num_queues)
283 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
284 gve_tx_add_to_block(priv, idx);
285
286 return 0;
287
288 abort_with_fifo:
289 if (!tx->raw_addressing)
290 gve_tx_fifo_release(priv, &tx->tx_fifo);
291 abort_with_qpl:
292 if (!tx->raw_addressing)
293 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
294 abort_with_desc:
295 dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
296 tx->desc = NULL;
297 abort_with_info:
298 vfree(tx->info);
299 tx->info = NULL;
300 return -ENOMEM;
301 }
302
gve_tx_alloc_rings(struct gve_priv * priv,int start_id,int num_rings)303 int gve_tx_alloc_rings(struct gve_priv *priv, int start_id, int num_rings)
304 {
305 int err = 0;
306 int i;
307
308 for (i = start_id; i < start_id + num_rings; i++) {
309 err = gve_tx_alloc_ring(priv, i);
310 if (err) {
311 netif_err(priv, drv, priv->dev,
312 "Failed to alloc tx ring=%d: err=%d\n",
313 i, err);
314 break;
315 }
316 }
317 /* Unallocate if there was an error */
318 if (err) {
319 int j;
320
321 for (j = start_id; j < i; j++)
322 gve_tx_free_ring(priv, j);
323 }
324 return err;
325 }
326
gve_tx_free_rings_gqi(struct gve_priv * priv,int start_id,int num_rings)327 void gve_tx_free_rings_gqi(struct gve_priv *priv, int start_id, int num_rings)
328 {
329 int i;
330
331 for (i = start_id; i < start_id + num_rings; i++)
332 gve_tx_free_ring(priv, i);
333 }
334
335 /* gve_tx_avail - Calculates the number of slots available in the ring
336 * @tx: tx ring to check
337 *
338 * Returns the number of slots available
339 *
340 * The capacity of the queue is mask + 1. We don't need to reserve an entry.
341 **/
gve_tx_avail(struct gve_tx_ring * tx)342 static inline u32 gve_tx_avail(struct gve_tx_ring *tx)
343 {
344 return tx->mask + 1 - (tx->req - tx->done);
345 }
346
gve_skb_fifo_bytes_required(struct gve_tx_ring * tx,struct sk_buff * skb)347 static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
348 struct sk_buff *skb)
349 {
350 int pad_bytes, align_hdr_pad;
351 int bytes;
352 int hlen;
353
354 hlen = skb_is_gso(skb) ? skb_checksum_start_offset(skb) + tcp_hdrlen(skb) :
355 min_t(int, GVE_GQ_TX_MIN_PKT_DESC_BYTES, skb->len);
356
357 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo,
358 hlen);
359 /* We need to take into account the header alignment padding. */
360 align_hdr_pad = L1_CACHE_ALIGN(hlen) - hlen;
361 bytes = align_hdr_pad + pad_bytes + skb->len;
362
363 return bytes;
364 }
365
366 /* The most descriptors we could need is MAX_SKB_FRAGS + 4 :
367 * 1 for each skb frag
368 * 1 for the skb linear portion
369 * 1 for when tcp hdr needs to be in separate descriptor
370 * 1 if the payload wraps to the beginning of the FIFO
371 * 1 for metadata descriptor
372 */
373 #define MAX_TX_DESC_NEEDED (MAX_SKB_FRAGS + 4)
gve_tx_unmap_buf(struct device * dev,struct gve_tx_buffer_state * info)374 static void gve_tx_unmap_buf(struct device *dev, struct gve_tx_buffer_state *info)
375 {
376 if (info->skb) {
377 dma_unmap_single(dev, dma_unmap_addr(info, dma),
378 dma_unmap_len(info, len),
379 DMA_TO_DEVICE);
380 dma_unmap_len_set(info, len, 0);
381 } else {
382 dma_unmap_page(dev, dma_unmap_addr(info, dma),
383 dma_unmap_len(info, len),
384 DMA_TO_DEVICE);
385 dma_unmap_len_set(info, len, 0);
386 }
387 }
388
389 /* Check if sufficient resources (descriptor ring space, FIFO space) are
390 * available to transmit the given number of bytes.
391 */
gve_can_tx(struct gve_tx_ring * tx,int bytes_required)392 static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required)
393 {
394 bool can_alloc = true;
395
396 if (!tx->raw_addressing)
397 can_alloc = gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required);
398
399 return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED && can_alloc);
400 }
401
402 static_assert(NAPI_POLL_WEIGHT >= MAX_TX_DESC_NEEDED);
403
404 /* Stops the queue if the skb cannot be transmitted. */
gve_maybe_stop_tx(struct gve_priv * priv,struct gve_tx_ring * tx,struct sk_buff * skb)405 static int gve_maybe_stop_tx(struct gve_priv *priv, struct gve_tx_ring *tx,
406 struct sk_buff *skb)
407 {
408 int bytes_required = 0;
409 u32 nic_done;
410 u32 to_do;
411 int ret;
412
413 if (!tx->raw_addressing)
414 bytes_required = gve_skb_fifo_bytes_required(tx, skb);
415
416 if (likely(gve_can_tx(tx, bytes_required)))
417 return 0;
418
419 ret = -EBUSY;
420 spin_lock(&tx->clean_lock);
421 nic_done = gve_tx_load_event_counter(priv, tx);
422 to_do = nic_done - tx->done;
423
424 /* Only try to clean if there is hope for TX */
425 if (to_do + gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED) {
426 if (to_do > 0) {
427 to_do = min_t(u32, to_do, NAPI_POLL_WEIGHT);
428 gve_clean_tx_done(priv, tx, to_do, false);
429 }
430 if (likely(gve_can_tx(tx, bytes_required)))
431 ret = 0;
432 }
433 if (ret) {
434 /* No space, so stop the queue */
435 tx->stop_queue++;
436 netif_tx_stop_queue(tx->netdev_txq);
437 }
438 spin_unlock(&tx->clean_lock);
439
440 return ret;
441 }
442
gve_tx_fill_pkt_desc(union gve_tx_desc * pkt_desc,u16 csum_offset,u8 ip_summed,bool is_gso,int l4_hdr_offset,u32 desc_cnt,u16 hlen,u64 addr,u16 pkt_len)443 static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
444 u16 csum_offset, u8 ip_summed, bool is_gso,
445 int l4_hdr_offset, u32 desc_cnt,
446 u16 hlen, u64 addr, u16 pkt_len)
447 {
448 /* l4_hdr_offset and csum_offset are in units of 16-bit words */
449 if (is_gso) {
450 pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
451 pkt_desc->pkt.l4_csum_offset = csum_offset >> 1;
452 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
453 } else if (likely(ip_summed == CHECKSUM_PARTIAL)) {
454 pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
455 pkt_desc->pkt.l4_csum_offset = csum_offset >> 1;
456 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
457 } else {
458 pkt_desc->pkt.type_flags = GVE_TXD_STD;
459 pkt_desc->pkt.l4_csum_offset = 0;
460 pkt_desc->pkt.l4_hdr_offset = 0;
461 }
462 pkt_desc->pkt.desc_cnt = desc_cnt;
463 pkt_desc->pkt.len = cpu_to_be16(pkt_len);
464 pkt_desc->pkt.seg_len = cpu_to_be16(hlen);
465 pkt_desc->pkt.seg_addr = cpu_to_be64(addr);
466 }
467
gve_tx_fill_mtd_desc(union gve_tx_desc * mtd_desc,struct sk_buff * skb)468 static void gve_tx_fill_mtd_desc(union gve_tx_desc *mtd_desc,
469 struct sk_buff *skb)
470 {
471 BUILD_BUG_ON(sizeof(mtd_desc->mtd) != sizeof(mtd_desc->pkt));
472
473 mtd_desc->mtd.type_flags = GVE_TXD_MTD | GVE_MTD_SUBTYPE_PATH;
474 mtd_desc->mtd.path_state = GVE_MTD_PATH_STATE_DEFAULT |
475 GVE_MTD_PATH_HASH_L4;
476 mtd_desc->mtd.path_hash = cpu_to_be32(skb->hash);
477 mtd_desc->mtd.reserved0 = 0;
478 mtd_desc->mtd.reserved1 = 0;
479 }
480
gve_tx_fill_seg_desc(union gve_tx_desc * seg_desc,u16 l3_offset,u16 gso_size,bool is_gso_v6,bool is_gso,u16 len,u64 addr)481 static void gve_tx_fill_seg_desc(union gve_tx_desc *seg_desc,
482 u16 l3_offset, u16 gso_size,
483 bool is_gso_v6, bool is_gso,
484 u16 len, u64 addr)
485 {
486 seg_desc->seg.type_flags = GVE_TXD_SEG;
487 if (is_gso) {
488 if (is_gso_v6)
489 seg_desc->seg.type_flags |= GVE_TXSF_IPV6;
490 seg_desc->seg.l3_offset = l3_offset >> 1;
491 seg_desc->seg.mss = cpu_to_be16(gso_size);
492 }
493 seg_desc->seg.seg_len = cpu_to_be16(len);
494 seg_desc->seg.seg_addr = cpu_to_be64(addr);
495 }
496
gve_dma_sync_for_device(struct device * dev,dma_addr_t * page_buses,u64 iov_offset,u64 iov_len)497 static void gve_dma_sync_for_device(struct device *dev, dma_addr_t *page_buses,
498 u64 iov_offset, u64 iov_len)
499 {
500 u64 last_page = (iov_offset + iov_len - 1) / PAGE_SIZE;
501 u64 first_page = iov_offset / PAGE_SIZE;
502 u64 page;
503
504 for (page = first_page; page <= last_page; page++)
505 dma_sync_single_for_device(dev, page_buses[page], PAGE_SIZE, DMA_TO_DEVICE);
506 }
507
gve_tx_add_skb_copy(struct gve_priv * priv,struct gve_tx_ring * tx,struct sk_buff * skb)508 static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, struct sk_buff *skb)
509 {
510 int pad_bytes, hlen, hdr_nfrags, payload_nfrags, l4_hdr_offset;
511 union gve_tx_desc *pkt_desc, *seg_desc;
512 struct gve_tx_buffer_state *info;
513 int mtd_desc_nr = !!skb->l4_hash;
514 bool is_gso = skb_is_gso(skb);
515 u32 idx = tx->req & tx->mask;
516 int payload_iov = 2;
517 int copy_offset;
518 u32 next_idx;
519 int i;
520
521 info = &tx->info[idx];
522 pkt_desc = &tx->desc[idx];
523
524 l4_hdr_offset = skb_checksum_start_offset(skb);
525 /* If the skb is gso, then we want the tcp header alone in the first segment
526 * otherwise we want the minimum required by the gVNIC spec.
527 */
528 hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) :
529 min_t(int, GVE_GQ_TX_MIN_PKT_DESC_BYTES, skb->len);
530
531 info->skb = skb;
532 /* We don't want to split the header, so if necessary, pad to the end
533 * of the fifo and then put the header at the beginning of the fifo.
534 */
535 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen);
536 hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes,
537 &info->iov[0]);
538 WARN(!hdr_nfrags, "hdr_nfrags should never be 0!");
539 payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,
540 &info->iov[payload_iov]);
541
542 gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed,
543 is_gso, l4_hdr_offset,
544 1 + mtd_desc_nr + payload_nfrags, hlen,
545 info->iov[hdr_nfrags - 1].iov_offset, skb->len);
546
547 skb_copy_bits(skb, 0,
548 tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,
549 hlen);
550 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,
551 info->iov[hdr_nfrags - 1].iov_offset,
552 info->iov[hdr_nfrags - 1].iov_len);
553 copy_offset = hlen;
554
555 if (mtd_desc_nr) {
556 next_idx = (tx->req + 1) & tx->mask;
557 gve_tx_fill_mtd_desc(&tx->desc[next_idx], skb);
558 }
559
560 for (i = payload_iov; i < payload_nfrags + payload_iov; i++) {
561 next_idx = (tx->req + 1 + mtd_desc_nr + i - payload_iov) & tx->mask;
562 seg_desc = &tx->desc[next_idx];
563
564 gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
565 skb_shinfo(skb)->gso_size,
566 skb_is_gso_v6(skb), is_gso,
567 info->iov[i].iov_len,
568 info->iov[i].iov_offset);
569
570 skb_copy_bits(skb, copy_offset,
571 tx->tx_fifo.base + info->iov[i].iov_offset,
572 info->iov[i].iov_len);
573 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,
574 info->iov[i].iov_offset,
575 info->iov[i].iov_len);
576 copy_offset += info->iov[i].iov_len;
577 }
578
579 return 1 + mtd_desc_nr + payload_nfrags;
580 }
581
gve_tx_add_skb_no_copy(struct gve_priv * priv,struct gve_tx_ring * tx,struct sk_buff * skb)582 static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
583 struct sk_buff *skb)
584 {
585 const struct skb_shared_info *shinfo = skb_shinfo(skb);
586 int hlen, num_descriptors, l4_hdr_offset;
587 union gve_tx_desc *pkt_desc, *mtd_desc, *seg_desc;
588 struct gve_tx_buffer_state *info;
589 int mtd_desc_nr = !!skb->l4_hash;
590 bool is_gso = skb_is_gso(skb);
591 u32 idx = tx->req & tx->mask;
592 u64 addr;
593 u32 len;
594 int i;
595
596 info = &tx->info[idx];
597 pkt_desc = &tx->desc[idx];
598
599 l4_hdr_offset = skb_checksum_start_offset(skb);
600 /* If the skb is gso, then we want only up to the tcp header in the first segment
601 * to efficiently replicate on each segment otherwise we want the linear portion
602 * of the skb (which will contain the checksum because skb->csum_start and
603 * skb->csum_offset are given relative to skb->head) in the first segment.
604 */
605 hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) : skb_headlen(skb);
606 len = skb_headlen(skb);
607
608 info->skb = skb;
609
610 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
611 if (unlikely(dma_mapping_error(tx->dev, addr))) {
612 tx->dma_mapping_error++;
613 goto drop;
614 }
615 dma_unmap_len_set(info, len, len);
616 dma_unmap_addr_set(info, dma, addr);
617
618 num_descriptors = 1 + shinfo->nr_frags;
619 if (hlen < len)
620 num_descriptors++;
621 if (mtd_desc_nr)
622 num_descriptors++;
623
624 gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed,
625 is_gso, l4_hdr_offset,
626 num_descriptors, hlen, addr, skb->len);
627
628 if (mtd_desc_nr) {
629 idx = (idx + 1) & tx->mask;
630 mtd_desc = &tx->desc[idx];
631 gve_tx_fill_mtd_desc(mtd_desc, skb);
632 }
633
634 if (hlen < len) {
635 /* For gso the rest of the linear portion of the skb needs to
636 * be in its own descriptor.
637 */
638 len -= hlen;
639 addr += hlen;
640 idx = (idx + 1) & tx->mask;
641 seg_desc = &tx->desc[idx];
642 gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
643 skb_shinfo(skb)->gso_size,
644 skb_is_gso_v6(skb), is_gso, len, addr);
645 }
646
647 for (i = 0; i < shinfo->nr_frags; i++) {
648 const skb_frag_t *frag = &shinfo->frags[i];
649
650 idx = (idx + 1) & tx->mask;
651 seg_desc = &tx->desc[idx];
652 len = skb_frag_size(frag);
653 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE);
654 if (unlikely(dma_mapping_error(tx->dev, addr))) {
655 tx->dma_mapping_error++;
656 goto unmap_drop;
657 }
658 tx->info[idx].skb = NULL;
659 dma_unmap_len_set(&tx->info[idx], len, len);
660 dma_unmap_addr_set(&tx->info[idx], dma, addr);
661
662 gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
663 skb_shinfo(skb)->gso_size,
664 skb_is_gso_v6(skb), is_gso, len, addr);
665 }
666
667 return num_descriptors;
668
669 unmap_drop:
670 i += num_descriptors - shinfo->nr_frags;
671 while (i--) {
672 /* Skip metadata descriptor, if set */
673 if (i == 1 && mtd_desc_nr == 1)
674 continue;
675 idx--;
676 gve_tx_unmap_buf(tx->dev, &tx->info[idx & tx->mask]);
677 }
678 drop:
679 tx->dropped_pkt++;
680 return 0;
681 }
682
gve_tx(struct sk_buff * skb,struct net_device * dev)683 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev)
684 {
685 struct gve_priv *priv = netdev_priv(dev);
686 struct gve_tx_ring *tx;
687 int nsegs;
688
689 WARN(skb_get_queue_mapping(skb) >= priv->tx_cfg.num_queues,
690 "skb queue index out of range");
691 tx = &priv->tx[skb_get_queue_mapping(skb)];
692 if (unlikely(gve_maybe_stop_tx(priv, tx, skb))) {
693 /* We need to ring the txq doorbell -- we have stopped the Tx
694 * queue for want of resources, but prior calls to gve_tx()
695 * may have added descriptors without ringing the doorbell.
696 */
697
698 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
699 return NETDEV_TX_BUSY;
700 }
701 if (tx->raw_addressing)
702 nsegs = gve_tx_add_skb_no_copy(priv, tx, skb);
703 else
704 nsegs = gve_tx_add_skb_copy(priv, tx, skb);
705
706 /* If the packet is getting sent, we need to update the skb */
707 if (nsegs) {
708 netdev_tx_sent_queue(tx->netdev_txq, skb->len);
709 skb_tx_timestamp(skb);
710 tx->req += nsegs;
711 } else {
712 dev_kfree_skb_any(skb);
713 }
714
715 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
716 return NETDEV_TX_OK;
717
718 /* Give packets to NIC. Even if this packet failed to send the doorbell
719 * might need to be rung because of xmit_more.
720 */
721 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
722 return NETDEV_TX_OK;
723 }
724
gve_tx_fill_xdp(struct gve_priv * priv,struct gve_tx_ring * tx,void * data,int len,void * frame_p,bool is_xsk)725 static int gve_tx_fill_xdp(struct gve_priv *priv, struct gve_tx_ring *tx,
726 void *data, int len, void *frame_p, bool is_xsk)
727 {
728 int pad, nfrags, ndescs, iovi, offset;
729 struct gve_tx_buffer_state *info;
730 u32 reqi = tx->req;
731
732 pad = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, len);
733 if (pad >= GVE_GQ_TX_MIN_PKT_DESC_BYTES)
734 pad = 0;
735 info = &tx->info[reqi & tx->mask];
736 info->xdp_frame = frame_p;
737 info->xdp.size = len;
738 info->xdp.is_xsk = is_xsk;
739
740 nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, pad + len,
741 &info->iov[0]);
742 iovi = pad > 0;
743 ndescs = nfrags - iovi;
744 offset = 0;
745
746 while (iovi < nfrags) {
747 if (!offset)
748 gve_tx_fill_pkt_desc(&tx->desc[reqi & tx->mask], 0,
749 CHECKSUM_NONE, false, 0, ndescs,
750 info->iov[iovi].iov_len,
751 info->iov[iovi].iov_offset, len);
752 else
753 gve_tx_fill_seg_desc(&tx->desc[reqi & tx->mask],
754 0, 0, false, false,
755 info->iov[iovi].iov_len,
756 info->iov[iovi].iov_offset);
757
758 memcpy(tx->tx_fifo.base + info->iov[iovi].iov_offset,
759 data + offset, info->iov[iovi].iov_len);
760 gve_dma_sync_for_device(&priv->pdev->dev,
761 tx->tx_fifo.qpl->page_buses,
762 info->iov[iovi].iov_offset,
763 info->iov[iovi].iov_len);
764 offset += info->iov[iovi].iov_len;
765 iovi++;
766 reqi++;
767 }
768
769 return ndescs;
770 }
771
gve_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)772 int gve_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
773 u32 flags)
774 {
775 struct gve_priv *priv = netdev_priv(dev);
776 struct gve_tx_ring *tx;
777 int i, err = 0, qid;
778
779 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
780 return -EINVAL;
781
782 qid = gve_xdp_tx_queue_id(priv,
783 smp_processor_id() % priv->num_xdp_queues);
784
785 tx = &priv->tx[qid];
786
787 spin_lock(&tx->xdp_lock);
788 for (i = 0; i < n; i++) {
789 err = gve_xdp_xmit_one(priv, tx, frames[i]->data,
790 frames[i]->len, frames[i]);
791 if (err)
792 break;
793 }
794
795 if (flags & XDP_XMIT_FLUSH)
796 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
797
798 spin_unlock(&tx->xdp_lock);
799
800 u64_stats_update_begin(&tx->statss);
801 tx->xdp_xmit += n;
802 tx->xdp_xmit_errors += n - i;
803 u64_stats_update_end(&tx->statss);
804
805 return i ? i : err;
806 }
807
gve_xdp_xmit_one(struct gve_priv * priv,struct gve_tx_ring * tx,void * data,int len,void * frame_p)808 int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx,
809 void *data, int len, void *frame_p)
810 {
811 int nsegs;
812
813 if (!gve_can_tx(tx, len + GVE_GQ_TX_MIN_PKT_DESC_BYTES - 1))
814 return -EBUSY;
815
816 nsegs = gve_tx_fill_xdp(priv, tx, data, len, frame_p, false);
817 tx->req += nsegs;
818
819 return 0;
820 }
821
822 #define GVE_TX_START_THRESH PAGE_SIZE
823
gve_clean_tx_done(struct gve_priv * priv,struct gve_tx_ring * tx,u32 to_do,bool try_to_wake)824 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
825 u32 to_do, bool try_to_wake)
826 {
827 struct gve_tx_buffer_state *info;
828 u64 pkts = 0, bytes = 0;
829 size_t space_freed = 0;
830 struct sk_buff *skb;
831 u32 idx;
832 int j;
833
834 for (j = 0; j < to_do; j++) {
835 idx = tx->done & tx->mask;
836 netif_info(priv, tx_done, priv->dev,
837 "[%d] %s: idx=%d (req=%u done=%u)\n",
838 tx->q_num, __func__, idx, tx->req, tx->done);
839 info = &tx->info[idx];
840 skb = info->skb;
841
842 /* Unmap the buffer */
843 if (tx->raw_addressing)
844 gve_tx_unmap_buf(tx->dev, info);
845 tx->done++;
846 /* Mark as free */
847 if (skb) {
848 info->skb = NULL;
849 bytes += skb->len;
850 pkts++;
851 dev_consume_skb_any(skb);
852 if (tx->raw_addressing)
853 continue;
854 space_freed += gve_tx_clear_buffer_state(info);
855 }
856 }
857
858 if (!tx->raw_addressing)
859 gve_tx_free_fifo(&tx->tx_fifo, space_freed);
860 u64_stats_update_begin(&tx->statss);
861 tx->bytes_done += bytes;
862 tx->pkt_done += pkts;
863 u64_stats_update_end(&tx->statss);
864 netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes);
865
866 /* start the queue if we've stopped it */
867 #ifndef CONFIG_BQL
868 /* Make sure that the doorbells are synced */
869 smp_mb();
870 #endif
871 if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) &&
872 likely(gve_can_tx(tx, GVE_TX_START_THRESH))) {
873 tx->wake_queue++;
874 netif_tx_wake_queue(tx->netdev_txq);
875 }
876
877 return pkts;
878 }
879
gve_tx_load_event_counter(struct gve_priv * priv,struct gve_tx_ring * tx)880 u32 gve_tx_load_event_counter(struct gve_priv *priv,
881 struct gve_tx_ring *tx)
882 {
883 u32 counter_index = be32_to_cpu(tx->q_resources->counter_index);
884 __be32 counter = READ_ONCE(priv->counter_array[counter_index]);
885
886 return be32_to_cpu(counter);
887 }
888
gve_xsk_tx(struct gve_priv * priv,struct gve_tx_ring * tx,int budget)889 static int gve_xsk_tx(struct gve_priv *priv, struct gve_tx_ring *tx,
890 int budget)
891 {
892 struct xdp_desc desc;
893 int sent = 0, nsegs;
894 void *data;
895
896 spin_lock(&tx->xdp_lock);
897 while (sent < budget) {
898 if (!gve_can_tx(tx, GVE_TX_START_THRESH))
899 goto out;
900
901 if (!xsk_tx_peek_desc(tx->xsk_pool, &desc)) {
902 tx->xdp_xsk_done = tx->xdp_xsk_wakeup;
903 goto out;
904 }
905
906 data = xsk_buff_raw_get_data(tx->xsk_pool, desc.addr);
907 nsegs = gve_tx_fill_xdp(priv, tx, data, desc.len, NULL, true);
908 tx->req += nsegs;
909 sent++;
910 }
911 out:
912 if (sent > 0) {
913 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
914 xsk_tx_release(tx->xsk_pool);
915 }
916 spin_unlock(&tx->xdp_lock);
917 return sent;
918 }
919
gve_xdp_poll(struct gve_notify_block * block,int budget)920 bool gve_xdp_poll(struct gve_notify_block *block, int budget)
921 {
922 struct gve_priv *priv = block->priv;
923 struct gve_tx_ring *tx = block->tx;
924 u32 nic_done;
925 bool repoll;
926 u32 to_do;
927
928 /* If budget is 0, do all the work */
929 if (budget == 0)
930 budget = INT_MAX;
931
932 /* Find out how much work there is to be done */
933 nic_done = gve_tx_load_event_counter(priv, tx);
934 to_do = min_t(u32, (nic_done - tx->done), budget);
935 gve_clean_xdp_done(priv, tx, to_do);
936 repoll = nic_done != tx->done;
937
938 if (tx->xsk_pool) {
939 int sent = gve_xsk_tx(priv, tx, budget);
940
941 u64_stats_update_begin(&tx->statss);
942 tx->xdp_xsk_sent += sent;
943 u64_stats_update_end(&tx->statss);
944 repoll |= (sent == budget);
945 if (xsk_uses_need_wakeup(tx->xsk_pool))
946 xsk_set_tx_need_wakeup(tx->xsk_pool);
947 }
948
949 /* If we still have work we want to repoll */
950 return repoll;
951 }
952
gve_tx_poll(struct gve_notify_block * block,int budget)953 bool gve_tx_poll(struct gve_notify_block *block, int budget)
954 {
955 struct gve_priv *priv = block->priv;
956 struct gve_tx_ring *tx = block->tx;
957 u32 nic_done;
958 u32 to_do;
959
960 /* If budget is 0, do all the work */
961 if (budget == 0)
962 budget = INT_MAX;
963
964 /* In TX path, it may try to clean completed pkts in order to xmit,
965 * to avoid cleaning conflict, use spin_lock(), it yields better
966 * concurrency between xmit/clean than netif's lock.
967 */
968 spin_lock(&tx->clean_lock);
969 /* Find out how much work there is to be done */
970 nic_done = gve_tx_load_event_counter(priv, tx);
971 to_do = min_t(u32, (nic_done - tx->done), budget);
972 gve_clean_tx_done(priv, tx, to_do, true);
973 spin_unlock(&tx->clean_lock);
974 /* If we still have work we want to repoll */
975 return nic_done != tx->done;
976 }
977
gve_tx_clean_pending(struct gve_priv * priv,struct gve_tx_ring * tx)978 bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx)
979 {
980 u32 nic_done = gve_tx_load_event_counter(priv, tx);
981
982 return nic_done != tx->done;
983 }
984