Lines Matching +full:prefetch +full:- +full:dma
1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
4 * Copyright (C) 2015-2021 Google, Inc.
16 dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) & in gve_rx_free_buffer() local
19 page_ref_sub(page_info->page, page_info->pagecnt_bias - 1); in gve_rx_free_buffer()
20 gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE); in gve_rx_free_buffer()
25 u32 slots = rx->mask + 1; in gve_rx_unfill_pages()
28 if (rx->data.raw_addressing) { in gve_rx_unfill_pages()
30 gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i], in gve_rx_unfill_pages()
31 &rx->data.data_ring[i]); in gve_rx_unfill_pages()
34 page_ref_sub(rx->data.page_info[i].page, in gve_rx_unfill_pages()
35 rx->data.page_info[i].pagecnt_bias - 1); in gve_rx_unfill_pages()
36 gve_unassign_qpl(priv, rx->data.qpl->id); in gve_rx_unfill_pages()
37 rx->data.qpl = NULL; in gve_rx_unfill_pages()
39 kvfree(rx->data.page_info); in gve_rx_unfill_pages()
40 rx->data.page_info = NULL; in gve_rx_unfill_pages()
45 struct gve_rx_ring *rx = &priv->rx[idx]; in gve_rx_free_ring()
46 struct device *dev = &priv->pdev->dev; in gve_rx_free_ring()
47 u32 slots = rx->mask + 1; in gve_rx_free_ring()
52 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt; in gve_rx_free_ring()
53 dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus); in gve_rx_free_ring()
54 rx->desc.desc_ring = NULL; in gve_rx_free_ring()
56 dma_free_coherent(dev, sizeof(*rx->q_resources), in gve_rx_free_ring()
57 rx->q_resources, rx->q_resources_bus); in gve_rx_free_ring()
58 rx->q_resources = NULL; in gve_rx_free_ring()
62 bytes = sizeof(*rx->data.data_ring) * slots; in gve_rx_free_ring()
63 dma_free_coherent(dev, bytes, rx->data.data_ring, in gve_rx_free_ring()
64 rx->data.data_bus); in gve_rx_free_ring()
65 rx->data.data_ring = NULL; in gve_rx_free_ring()
66 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx); in gve_rx_free_ring()
72 page_info->page = page; in gve_setup_rx_buffer()
73 page_info->page_offset = 0; in gve_setup_rx_buffer()
74 page_info->page_address = page_address(page); in gve_setup_rx_buffer()
77 page_ref_add(page, INT_MAX - 1); in gve_setup_rx_buffer()
78 page_info->pagecnt_bias = INT_MAX; in gve_setup_rx_buffer()
86 dma_addr_t dma; in gve_rx_alloc_buffer() local
89 err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE, in gve_rx_alloc_buffer()
94 gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr); in gve_rx_alloc_buffer()
100 struct gve_priv *priv = rx->gve; in gve_prefill_rx_pages()
108 slots = rx->mask + 1; in gve_prefill_rx_pages()
110 rx->data.page_info = kvzalloc(slots * in gve_prefill_rx_pages()
111 sizeof(*rx->data.page_info), GFP_KERNEL); in gve_prefill_rx_pages()
112 if (!rx->data.page_info) in gve_prefill_rx_pages()
113 return -ENOMEM; in gve_prefill_rx_pages()
115 if (!rx->data.raw_addressing) { in gve_prefill_rx_pages()
116 rx->data.qpl = gve_assign_rx_qpl(priv); in gve_prefill_rx_pages()
117 if (!rx->data.qpl) { in gve_prefill_rx_pages()
118 kvfree(rx->data.page_info); in gve_prefill_rx_pages()
119 rx->data.page_info = NULL; in gve_prefill_rx_pages()
120 return -ENOMEM; in gve_prefill_rx_pages()
124 if (!rx->data.raw_addressing) { in gve_prefill_rx_pages()
125 struct page *page = rx->data.qpl->pages[i]; in gve_prefill_rx_pages()
128 gve_setup_rx_buffer(&rx->data.page_info[i], addr, page, in gve_prefill_rx_pages()
129 &rx->data.data_ring[i].qpl_offset); in gve_prefill_rx_pages()
132 err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i], in gve_prefill_rx_pages()
133 &rx->data.data_ring[i]); in gve_prefill_rx_pages()
140 while (i--) in gve_prefill_rx_pages()
141 gve_rx_free_buffer(&priv->pdev->dev, in gve_prefill_rx_pages()
142 &rx->data.page_info[i], in gve_prefill_rx_pages()
143 &rx->data.data_ring[i]); in gve_prefill_rx_pages()
149 ctx->curr_frag_cnt = 0; in gve_rx_ctx_clear()
150 ctx->total_expected_size = 0; in gve_rx_ctx_clear()
151 ctx->expected_frag_cnt = 0; in gve_rx_ctx_clear()
152 ctx->skb_head = NULL; in gve_rx_ctx_clear()
153 ctx->skb_tail = NULL; in gve_rx_ctx_clear()
154 ctx->reuse_frags = false; in gve_rx_ctx_clear()
159 struct gve_rx_ring *rx = &priv->rx[idx]; in gve_rx_alloc_ring()
160 struct device *hdev = &priv->pdev->dev; in gve_rx_alloc_ring()
166 netif_dbg(priv, drv, priv->dev, "allocating rx ring\n"); in gve_rx_alloc_ring()
170 rx->gve = priv; in gve_rx_alloc_ring()
171 rx->q_num = idx; in gve_rx_alloc_ring()
173 slots = priv->rx_data_slot_cnt; in gve_rx_alloc_ring()
174 rx->mask = slots - 1; in gve_rx_alloc_ring()
175 rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT; in gve_rx_alloc_ring()
178 bytes = sizeof(*rx->data.data_ring) * slots; in gve_rx_alloc_ring()
179 rx->data.data_ring = dma_alloc_coherent(hdev, bytes, in gve_rx_alloc_ring()
180 &rx->data.data_bus, in gve_rx_alloc_ring()
182 if (!rx->data.data_ring) in gve_rx_alloc_ring()
183 return -ENOMEM; in gve_rx_alloc_ring()
186 err = -ENOMEM; in gve_rx_alloc_ring()
189 rx->fill_cnt = filled_pages; in gve_rx_alloc_ring()
194 rx->q_resources = in gve_rx_alloc_ring()
196 sizeof(*rx->q_resources), in gve_rx_alloc_ring()
197 &rx->q_resources_bus, in gve_rx_alloc_ring()
199 if (!rx->q_resources) { in gve_rx_alloc_ring()
200 err = -ENOMEM; in gve_rx_alloc_ring()
203 netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx, in gve_rx_alloc_ring()
204 (unsigned long)rx->data.data_bus); in gve_rx_alloc_ring()
207 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt; in gve_rx_alloc_ring()
210 err = -EIO; in gve_rx_alloc_ring()
214 rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus, in gve_rx_alloc_ring()
216 if (!rx->desc.desc_ring) { in gve_rx_alloc_ring()
217 err = -ENOMEM; in gve_rx_alloc_ring()
220 rx->cnt = 0; in gve_rx_alloc_ring()
221 rx->db_threshold = priv->rx_desc_cnt / 2; in gve_rx_alloc_ring()
222 rx->desc.seqno = 1; in gve_rx_alloc_ring()
224 /* Allocating half-page buffers allows page-flipping which is faster in gve_rx_alloc_ring()
227 rx->packet_buffer_size = PAGE_SIZE / 2; in gve_rx_alloc_ring()
228 gve_rx_ctx_clear(&rx->ctx); in gve_rx_alloc_ring()
234 dma_free_coherent(hdev, sizeof(*rx->q_resources), in gve_rx_alloc_ring()
235 rx->q_resources, rx->q_resources_bus); in gve_rx_alloc_ring()
236 rx->q_resources = NULL; in gve_rx_alloc_ring()
240 bytes = sizeof(*rx->data.data_ring) * slots; in gve_rx_alloc_ring()
241 dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus); in gve_rx_alloc_ring()
242 rx->data.data_ring = NULL; in gve_rx_alloc_ring()
252 for (i = 0; i < priv->rx_cfg.num_queues; i++) { in gve_rx_alloc_rings()
255 netif_err(priv, drv, priv->dev, in gve_rx_alloc_rings()
275 for (i = 0; i < priv->rx_cfg.num_queues; i++) in gve_rx_free_rings_gqi()
281 u32 db_idx = be32_to_cpu(rx->q_resources->db_index); in gve_rx_write_doorbell()
283 iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]); in gve_rx_write_doorbell()
297 return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0; in gve_rx_ctx_padding()
305 u32 offset = page_info->page_offset + gve_rx_ctx_padding(ctx); in gve_rx_add_frags()
308 if (!ctx->skb_head) in gve_rx_add_frags()
309 ctx->skb_head = napi_get_frags(napi); in gve_rx_add_frags()
311 if (unlikely(!ctx->skb_head)) in gve_rx_add_frags()
314 skb = ctx->skb_head; in gve_rx_add_frags()
315 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page, in gve_rx_add_frags()
326 page_info->page_offset ^= PAGE_SIZE / 2; in gve_rx_flip_buff()
332 int pagecount = page_count(page_info->page); in gve_rx_can_recycle_buffer()
334 /* This page is not being used by any SKBs - reuse */ in gve_rx_can_recycle_buffer()
335 if (pagecount == page_info->pagecnt_bias) in gve_rx_can_recycle_buffer()
337 /* This page is still being used by an SKB - we can't reuse */ in gve_rx_can_recycle_buffer()
338 else if (pagecount > page_info->pagecnt_bias) in gve_rx_can_recycle_buffer()
340 WARN(pagecount < page_info->pagecnt_bias, in gve_rx_can_recycle_buffer()
342 return -1; in gve_rx_can_recycle_buffer()
372 struct gve_rx_ctx *ctx = &rx->ctx; in gve_rx_qpl()
380 if (ctx->reuse_frags) { in gve_rx_qpl()
381 skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx); in gve_rx_qpl()
386 gve_rx_flip_buff(page_info, &data_slot->qpl_offset); in gve_rx_qpl()
393 u64_stats_update_begin(&rx->statss); in gve_rx_qpl()
394 rx->rx_frag_copy_cnt++; in gve_rx_qpl()
395 u64_stats_update_end(&rx->statss); in gve_rx_qpl()
404 return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx); in gve_rx_get_fragment_size()
409 bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false; in gve_rx_ctx_init()
412 struct gve_priv *priv = rx->gve; in gve_rx_ctx_init()
413 u32 idx = rx->cnt & rx->mask; in gve_rx_ctx_init()
429 desc = &rx->desc.desc_ring[idx]; in gve_rx_ctx_init()
430 desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error; in gve_rx_ctx_init()
431 if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) { in gve_rx_ctx_init()
433 netdev_warn(priv->dev, in gve_rx_ctx_init()
435 rx->desc.seqno, GVE_SEQNO(desc->flags_seq)); in gve_rx_ctx_init()
437 frag_size = be16_to_cpu(desc->len); in gve_rx_ctx_init()
439 if (frag_size > rx->packet_buffer_size) { in gve_rx_ctx_init()
441 netdev_warn(priv->dev, in gve_rx_ctx_init()
443 rx->packet_buffer_size, be16_to_cpu(desc->len)); in gve_rx_ctx_init()
445 page_info = &rx->data.page_info[idx]; in gve_rx_ctx_init()
451 idx = (idx + 1) & rx->mask; in gve_rx_ctx_init()
452 rx->desc.seqno = gve_next_seqno(rx->desc.seqno); in gve_rx_ctx_init()
453 } while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq)); in gve_rx_ctx_init()
455 prefetch(rx->desc.desc_ring + idx); in gve_rx_ctx_init()
457 ctx->curr_frag_cnt = 0; in gve_rx_ctx_init()
458 ctx->total_expected_size = packet_size - GVE_RX_PAD; in gve_rx_ctx_init()
459 ctx->expected_frag_cnt = n_frags; in gve_rx_ctx_init()
460 ctx->skb_head = NULL; in gve_rx_ctx_init()
461 ctx->reuse_frags = reuse_frags; in gve_rx_ctx_init()
463 if (ctx->expected_frag_cnt > 1) { in gve_rx_ctx_init()
464 u64_stats_update_begin(&rx->statss); in gve_rx_ctx_init()
465 rx->rx_cont_packet_cnt++; in gve_rx_ctx_init()
466 u64_stats_update_end(&rx->statss); in gve_rx_ctx_init()
468 if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) { in gve_rx_ctx_init()
469 u64_stats_update_begin(&rx->statss); in gve_rx_ctx_init()
470 rx->rx_copied_pkt++; in gve_rx_ctx_init()
471 u64_stats_update_end(&rx->statss); in gve_rx_ctx_init()
480 u64_stats_update_begin(&rx->statss); in gve_rx_ctx_init()
481 rx->rx_desc_err_dropped_pkt++; in gve_rx_ctx_init()
482 u64_stats_update_end(&rx->statss); in gve_rx_ctx_init()
492 struct net_device *netdev = priv->dev; in gve_rx_skb()
493 struct gve_rx_ctx *ctx = &rx->ctx; in gve_rx_skb()
496 if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) { in gve_rx_skb()
500 u64_stats_update_begin(&rx->statss); in gve_rx_skb()
501 rx->rx_copied_pkt++; in gve_rx_skb()
502 rx->rx_frag_copy_cnt++; in gve_rx_skb()
503 rx->rx_copybreak_pkt++; in gve_rx_skb()
504 u64_stats_update_end(&rx->statss); in gve_rx_skb()
507 if (rx->data.raw_addressing) { in gve_rx_skb()
514 page_info->can_flip = recycle; in gve_rx_skb()
515 if (page_info->can_flip) { in gve_rx_skb()
516 u64_stats_update_begin(&rx->statss); in gve_rx_skb()
517 rx->rx_frag_flip_cnt++; in gve_rx_skb()
518 u64_stats_update_end(&rx->statss); in gve_rx_skb()
520 skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev, in gve_rx_skb()
523 rx->packet_buffer_size, ctx); in gve_rx_skb()
525 if (ctx->reuse_frags) { in gve_rx_skb()
526 u64_stats_update_begin(&rx->statss); in gve_rx_skb()
527 rx->rx_frag_flip_cnt++; in gve_rx_skb()
528 u64_stats_update_end(&rx->statss); in gve_rx_skb()
530 skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx, in gve_rx_skb()
541 struct gve_rx_ctx *ctx = &rx->ctx; in gve_rx()
543 struct gve_priv *priv = rx->gve; in gve_rx()
554 idx = rx->cnt & rx->mask; in gve_rx()
555 first_desc = &rx->desc.desc_ring[idx]; in gve_rx()
557 napi = &priv->ntfy_blocks[rx->ntfy_id].napi; in gve_rx()
562 while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) { in gve_rx()
563 /* Prefetch two packet buffers ahead, we will need it soon. */ in gve_rx()
564 page_info = &rx->data.page_info[(idx + 2) & rx->mask]; in gve_rx()
565 va = page_info->page_address + page_info->page_offset; in gve_rx()
567 prefetch(page_info->page); /* Kernel page struct. */ in gve_rx()
568 prefetch(va); /* Packet header. */ in gve_rx()
569 prefetch(va + 64); /* Next cacheline too. */ in gve_rx()
573 page_info = &rx->data.page_info[idx]; in gve_rx()
574 data_slot = &rx->data.data_ring[idx]; in gve_rx()
575 page_bus = rx->data.raw_addressing ? in gve_rx()
576 be64_to_cpu(data_slot->addr) - page_info->page_offset : in gve_rx()
577 rx->data.qpl->page_buses[idx]; in gve_rx()
578 dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE); in gve_rx()
582 u64_stats_update_begin(&rx->statss); in gve_rx()
583 rx->rx_skb_alloc_fail++; in gve_rx()
584 u64_stats_update_end(&rx->statss); in gve_rx()
588 ctx->curr_frag_cnt++; in gve_rx()
589 rx->cnt++; in gve_rx()
590 idx = rx->cnt & rx->mask; in gve_rx()
592 desc = &rx->desc.desc_ring[idx]; in gve_rx()
597 if (first_desc->csum) in gve_rx()
598 skb->ip_summed = CHECKSUM_COMPLETE; in gve_rx()
600 skb->ip_summed = CHECKSUM_NONE; in gve_rx()
601 skb->csum = csum_unfold(first_desc->csum); in gve_rx()
606 gve_needs_rss(first_desc->flags_seq)) in gve_rx()
607 skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash), in gve_rx()
608 gve_rss_type(first_desc->flags_seq)); in gve_rx()
610 *packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0); in gve_rx()
612 skb_record_rx_queue(skb, rx->q_num); in gve_rx()
622 if (napi->skb) in gve_rx()
625 *work_done = ctx->expected_frag_cnt; in gve_rx()
626 while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) { in gve_rx()
627 rx->cnt++; in gve_rx()
628 ctx->curr_frag_cnt++; in gve_rx()
640 next_idx = rx->cnt & rx->mask; in gve_rx_work_pending()
641 desc = rx->desc.desc_ring + next_idx; in gve_rx_work_pending()
643 flags_seq = desc->flags_seq; in gve_rx_work_pending()
645 return (GVE_SEQNO(flags_seq) == rx->desc.seqno); in gve_rx_work_pending()
650 int refill_target = rx->mask + 1; in gve_rx_refill_buffers()
651 u32 fill_cnt = rx->fill_cnt; in gve_rx_refill_buffers()
653 while (fill_cnt - rx->cnt < refill_target) { in gve_rx_refill_buffers()
655 u32 idx = fill_cnt & rx->mask; in gve_rx_refill_buffers()
657 page_info = &rx->data.page_info[idx]; in gve_rx_refill_buffers()
658 if (page_info->can_flip) { in gve_rx_refill_buffers()
663 &rx->data.data_ring[idx]; in gve_rx_refill_buffers()
665 gve_rx_flip_buff(page_info, &data_slot->addr); in gve_rx_refill_buffers()
666 page_info->can_flip = 0; in gve_rx_refill_buffers()
671 * Flipping is unnecessary here - if the networking stack still in gve_rx_refill_buffers()
678 if (!rx->data.raw_addressing) in gve_rx_refill_buffers()
683 /* We can't reuse the buffer - alloc a new one*/ in gve_rx_refill_buffers()
685 &rx->data.data_ring[idx]; in gve_rx_refill_buffers()
686 struct device *dev = &priv->pdev->dev; in gve_rx_refill_buffers()
688 page_info->page = NULL; in gve_rx_refill_buffers()
691 u64_stats_update_begin(&rx->statss); in gve_rx_refill_buffers()
692 rx->rx_buf_alloc_fail++; in gve_rx_refill_buffers()
693 u64_stats_update_end(&rx->statss); in gve_rx_refill_buffers()
700 rx->fill_cnt = fill_cnt; in gve_rx_refill_buffers()
708 struct gve_priv *priv = rx->gve; in gve_clean_rx_done()
709 u32 idx = rx->cnt & rx->mask; in gve_clean_rx_done()
713 desc = &rx->desc.desc_ring[idx]; in gve_clean_rx_done()
714 while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) && in gve_clean_rx_done()
720 netif_info(priv, rx_status, priv->dev, in gve_clean_rx_done()
721 "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n", in gve_clean_rx_done()
722 rx->q_num, idx, desc, desc->flags_seq); in gve_clean_rx_done()
723 netif_info(priv, rx_status, priv->dev, in gve_clean_rx_done()
724 "[%d] seqno=%d rx->desc.seqno=%d\n", in gve_clean_rx_done()
725 rx->q_num, GVE_SEQNO(desc->flags_seq), in gve_clean_rx_done()
726 rx->desc.seqno); in gve_clean_rx_done()
734 idx = rx->cnt & rx->mask; in gve_clean_rx_done()
735 desc = &rx->desc.desc_ring[idx]; in gve_clean_rx_done()
739 if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold) in gve_clean_rx_done()
743 u64_stats_update_begin(&rx->statss); in gve_clean_rx_done()
744 rx->rpackets += ok_packet_cnt; in gve_clean_rx_done()
745 rx->rbytes += bytes; in gve_clean_rx_done()
746 u64_stats_update_end(&rx->statss); in gve_clean_rx_done()
750 if (!rx->data.raw_addressing) { in gve_clean_rx_done()
752 rx->fill_cnt += work_done; in gve_clean_rx_done()
753 } else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) { in gve_clean_rx_done()
763 if (rx->fill_cnt - rx->cnt <= rx->db_threshold) { in gve_clean_rx_done()
775 struct gve_rx_ring *rx = block->rx; in gve_rx_poll()
779 feat = block->napi.dev->features; in gve_rx_poll()