Lines Matching +full:queue +full:- +full:rx
4 * Copyright (c) 2002-2005, K A Fraser
77 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb))
86 /* Minimum number of Rx slots (includes slot for GSO metadata). */
89 /* Queue name is interface name with "-qNNN" appended */
92 /* IRQ name is queue name with "-tx" or "-rx" appended */
106 unsigned int id; /* Queue ID, 0-based */
107 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
120 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
121 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
128 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
142 struct xen_netif_rx_front_ring rx; member
161 /* Multi-queue support */
179 struct xen_netif_rx_response rx; member
180 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
207 return idx & (NET_RX_RING_SIZE - 1); in xennet_rxidx()
210 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, in xennet_get_rx_skb() argument
214 struct sk_buff *skb = queue->rx_skbs[i]; in xennet_get_rx_skb()
215 queue->rx_skbs[i] = NULL; in xennet_get_rx_skb()
219 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, in xennet_get_rx_ref() argument
223 grant_ref_t ref = queue->grant_rx_ref[i]; in xennet_get_rx_ref()
224 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_get_rx_ref()
234 return dev->features & NETIF_F_SG; in xennet_can_sg()
240 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); in rx_refill_timeout() local
241 napi_schedule(&queue->napi); in rx_refill_timeout()
244 static int netfront_tx_slot_available(struct netfront_queue *queue) in netfront_tx_slot_available() argument
246 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < in netfront_tx_slot_available()
247 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); in netfront_tx_slot_available()
250 static void xennet_maybe_wake_tx(struct netfront_queue *queue) in xennet_maybe_wake_tx() argument
252 struct net_device *dev = queue->info->netdev; in xennet_maybe_wake_tx()
253 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); in xennet_maybe_wake_tx()
256 netfront_tx_slot_available(queue) && in xennet_maybe_wake_tx()
258 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_maybe_wake_tx()
262 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) in xennet_alloc_one_rx_buffer() argument
267 skb = __netdev_alloc_skb(queue->info->netdev, in xennet_alloc_one_rx_buffer()
273 page = page_pool_dev_alloc_pages(queue->page_pool); in xennet_alloc_one_rx_buffer()
282 skb->dev = queue->info->netdev; in xennet_alloc_one_rx_buffer()
288 static void xennet_alloc_rx_buffers(struct netfront_queue *queue) in xennet_alloc_rx_buffers() argument
290 RING_IDX req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
294 if (unlikely(!netif_carrier_ok(queue->info->netdev))) in xennet_alloc_rx_buffers()
297 for (req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
298 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; in xennet_alloc_rx_buffers()
306 skb = xennet_alloc_one_rx_buffer(queue); in xennet_alloc_rx_buffers()
308 err = -ENOMEM; in xennet_alloc_rx_buffers()
314 BUG_ON(queue->rx_skbs[id]); in xennet_alloc_rx_buffers()
315 queue->rx_skbs[id] = skb; in xennet_alloc_rx_buffers()
317 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); in xennet_alloc_rx_buffers()
319 queue->grant_rx_ref[id] = ref; in xennet_alloc_rx_buffers()
321 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_alloc_rx_buffers()
323 req = RING_GET_REQUEST(&queue->rx, req_prod); in xennet_alloc_rx_buffers()
325 queue->info->xbdev->otherend_id, in xennet_alloc_rx_buffers()
328 req->id = id; in xennet_alloc_rx_buffers()
329 req->gref = ref; in xennet_alloc_rx_buffers()
332 queue->rx.req_prod_pvt = req_prod; in xennet_alloc_rx_buffers()
339 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || in xennet_alloc_rx_buffers()
341 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); in xennet_alloc_rx_buffers()
345 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); in xennet_alloc_rx_buffers()
347 notify_remote_via_irq(queue->rx_irq); in xennet_alloc_rx_buffers()
353 unsigned int num_queues = dev->real_num_tx_queues; in xennet_open()
355 struct netfront_queue *queue = NULL; in xennet_open() local
357 if (!np->queues || np->broken) in xennet_open()
358 return -ENODEV; in xennet_open()
361 queue = &np->queues[i]; in xennet_open()
362 napi_enable(&queue->napi); in xennet_open()
364 spin_lock_bh(&queue->rx_lock); in xennet_open()
366 xennet_alloc_rx_buffers(queue); in xennet_open()
367 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; in xennet_open()
368 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) in xennet_open()
369 napi_schedule(&queue->napi); in xennet_open()
371 spin_unlock_bh(&queue->rx_lock); in xennet_open()
379 static void xennet_tx_buf_gc(struct netfront_queue *queue) in xennet_tx_buf_gc() argument
385 const struct device *dev = &queue->info->netdev->dev; in xennet_tx_buf_gc()
387 BUG_ON(!netif_carrier_ok(queue->info->netdev)); in xennet_tx_buf_gc()
390 prod = queue->tx.sring->rsp_prod; in xennet_tx_buf_gc()
391 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { in xennet_tx_buf_gc()
393 prod - queue->tx.rsp_cons); in xennet_tx_buf_gc()
398 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { in xennet_tx_buf_gc()
401 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); in xennet_tx_buf_gc()
406 if (id >= RING_SIZE(&queue->tx)) { in xennet_tx_buf_gc()
412 if (queue->tx_link[id] != TX_PENDING) { in xennet_tx_buf_gc()
418 queue->tx_link[id] = TX_LINK_NONE; in xennet_tx_buf_gc()
419 skb = queue->tx_skbs[id]; in xennet_tx_buf_gc()
420 queue->tx_skbs[id] = NULL; in xennet_tx_buf_gc()
422 queue->grant_tx_ref[id]) != 0)) { in xennet_tx_buf_gc()
428 queue->grant_tx_ref[id], GNTMAP_readonly); in xennet_tx_buf_gc()
430 &queue->gref_tx_head, queue->grant_tx_ref[id]); in xennet_tx_buf_gc()
431 queue->grant_tx_ref[id] = GRANT_INVALID_REF; in xennet_tx_buf_gc()
432 queue->grant_tx_page[id] = NULL; in xennet_tx_buf_gc()
433 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); in xennet_tx_buf_gc()
437 queue->tx.rsp_cons = prod; in xennet_tx_buf_gc()
439 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); in xennet_tx_buf_gc()
442 xennet_maybe_wake_tx(queue); in xennet_tx_buf_gc()
447 queue->info->broken = true; in xennet_tx_buf_gc()
452 struct netfront_queue *queue; member
468 struct page *page = info->page; in xennet_tx_setup_grant()
469 struct netfront_queue *queue = info->queue; in xennet_tx_setup_grant() local
470 struct sk_buff *skb = info->skb; in xennet_tx_setup_grant()
472 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); in xennet_tx_setup_grant()
473 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_tx_setup_grant()
474 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); in xennet_tx_setup_grant()
477 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, in xennet_tx_setup_grant()
480 queue->tx_skbs[id] = skb; in xennet_tx_setup_grant()
481 queue->grant_tx_page[id] = page; in xennet_tx_setup_grant()
482 queue->grant_tx_ref[id] = ref; in xennet_tx_setup_grant()
484 info->tx_local.id = id; in xennet_tx_setup_grant()
485 info->tx_local.gref = ref; in xennet_tx_setup_grant()
486 info->tx_local.offset = offset; in xennet_tx_setup_grant()
487 info->tx_local.size = len; in xennet_tx_setup_grant()
488 info->tx_local.flags = 0; in xennet_tx_setup_grant()
490 *tx = info->tx_local; in xennet_tx_setup_grant()
493 * Put the request in the pending queue, it will be set to be pending in xennet_tx_setup_grant()
496 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); in xennet_tx_setup_grant()
498 info->tx = tx; in xennet_tx_setup_grant()
499 info->size += info->tx_local.size; in xennet_tx_setup_grant()
506 info->size = 0; in xennet_make_first_txreq()
508 gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info); in xennet_make_first_txreq()
510 return info->tx; in xennet_make_first_txreq()
518 info->tx->flags |= XEN_NETTXF_more_data; in xennet_make_one_txreq()
519 skb_get(info->skb); in xennet_make_one_txreq()
533 info->page = page; in xennet_make_txreqs()
534 info->size = 0; in xennet_make_txreqs()
542 len -= info->size; in xennet_make_txreqs()
552 int i, frags = skb_shinfo(skb)->nr_frags; in xennet_count_skb_slots()
555 slots = gnttab_count_grant(offset_in_page(skb->data), in xennet_count_skb_slots()
559 skb_frag_t *frag = skb_shinfo(skb)->frags + i; in xennet_count_skb_slots()
575 unsigned int num_queues = dev->real_num_tx_queues; in xennet_select_queue()
579 /* First, check if there is only one queue */ in xennet_select_queue()
590 static void xennet_mark_tx_pending(struct netfront_queue *queue) in xennet_mark_tx_pending() argument
594 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != in xennet_mark_tx_pending()
596 queue->tx_link[i] = TX_PENDING; in xennet_mark_tx_pending()
600 struct netfront_queue *queue, in xennet_xdp_xmit_one() argument
604 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); in xennet_xdp_xmit_one()
606 .queue = queue, in xennet_xdp_xmit_one()
608 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one()
613 offset_in_page(xdpf->data), in xennet_xdp_xmit_one()
614 xdpf->len); in xennet_xdp_xmit_one()
616 xennet_mark_tx_pending(queue); in xennet_xdp_xmit_one()
618 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_xdp_xmit_one()
620 notify_remote_via_irq(queue->tx_irq); in xennet_xdp_xmit_one()
622 u64_stats_update_begin(&tx_stats->syncp); in xennet_xdp_xmit_one()
623 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one()
624 tx_stats->packets++; in xennet_xdp_xmit_one()
625 u64_stats_update_end(&tx_stats->syncp); in xennet_xdp_xmit_one()
627 xennet_tx_buf_gc(queue); in xennet_xdp_xmit_one()
635 unsigned int num_queues = dev->real_num_tx_queues; in xennet_xdp_xmit()
637 struct netfront_queue *queue = NULL; in xennet_xdp_xmit() local
642 if (unlikely(np->broken)) in xennet_xdp_xmit()
643 return -ENODEV; in xennet_xdp_xmit()
645 return -EINVAL; in xennet_xdp_xmit()
647 queue = &np->queues[smp_processor_id() % num_queues]; in xennet_xdp_xmit()
649 spin_lock_irqsave(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
655 if (xennet_xdp_xmit_one(dev, queue, xdpf)) in xennet_xdp_xmit()
659 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
670 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); in xennet_start_xmit()
679 struct netfront_queue *queue = NULL; in xennet_start_xmit() local
681 unsigned int num_queues = dev->real_num_tx_queues; in xennet_start_xmit()
688 if (unlikely(np->broken)) in xennet_start_xmit()
690 /* Determine which queue to transmit this SKB on */ in xennet_start_xmit()
692 queue = &np->queues[queue_index]; in xennet_start_xmit()
694 /* If skb->len is too big for wire format, drop skb and alert in xennet_start_xmit()
697 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { in xennet_start_xmit()
699 "xennet: skb->len = %u, too big for wire format\n", in xennet_start_xmit()
700 skb->len); in xennet_start_xmit()
707 slots, skb->len); in xennet_start_xmit()
712 page = virt_to_page(skb->data); in xennet_start_xmit()
713 offset = offset_in_page(skb->data); in xennet_start_xmit()
718 if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { in xennet_start_xmit()
724 page = virt_to_page(skb->data); in xennet_start_xmit()
725 offset = offset_in_page(skb->data); in xennet_start_xmit()
730 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_start_xmit()
735 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
740 info.queue = queue; in xennet_start_xmit()
749 len -= info.tx_local.size; in xennet_start_xmit()
751 if (skb->ip_summed == CHECKSUM_PARTIAL) in xennet_start_xmit()
753 first_tx->flags |= XEN_NETTXF_csum_blank | in xennet_start_xmit()
755 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) in xennet_start_xmit()
757 first_tx->flags |= XEN_NETTXF_data_validated; in xennet_start_xmit()
760 if (skb_shinfo(skb)->gso_size) { in xennet_start_xmit()
764 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_start_xmit()
766 first_tx->flags |= XEN_NETTXF_extra_info; in xennet_start_xmit()
768 gso->u.gso.size = skb_shinfo(skb)->gso_size; in xennet_start_xmit()
769 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? in xennet_start_xmit()
772 gso->u.gso.pad = 0; in xennet_start_xmit()
773 gso->u.gso.features = 0; in xennet_start_xmit()
775 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; in xennet_start_xmit()
776 gso->flags = 0; in xennet_start_xmit()
783 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in xennet_start_xmit()
784 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in xennet_start_xmit()
791 first_tx->size = skb->len; in xennet_start_xmit()
796 xennet_mark_tx_pending(queue); in xennet_start_xmit()
798 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_start_xmit()
800 notify_remote_via_irq(queue->tx_irq); in xennet_start_xmit()
802 u64_stats_update_begin(&tx_stats->syncp); in xennet_start_xmit()
803 tx_stats->bytes += skb->len; in xennet_start_xmit()
804 tx_stats->packets++; in xennet_start_xmit()
805 u64_stats_update_end(&tx_stats->syncp); in xennet_start_xmit()
808 xennet_tx_buf_gc(queue); in xennet_start_xmit()
810 if (!netfront_tx_slot_available(queue)) in xennet_start_xmit()
811 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_start_xmit()
813 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
818 dev->stats.tx_dropped++; in xennet_start_xmit()
826 unsigned int num_queues = dev->real_num_tx_queues; in xennet_close()
828 struct netfront_queue *queue; in xennet_close() local
829 netif_tx_stop_all_queues(np->netdev); in xennet_close()
831 queue = &np->queues[i]; in xennet_close()
832 napi_disable(&queue->napi); in xennet_close()
837 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, in xennet_move_rx_slot() argument
840 int new = xennet_rxidx(queue->rx.req_prod_pvt); in xennet_move_rx_slot()
842 BUG_ON(queue->rx_skbs[new]); in xennet_move_rx_slot()
843 queue->rx_skbs[new] = skb; in xennet_move_rx_slot()
844 queue->grant_rx_ref[new] = ref; in xennet_move_rx_slot()
845 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; in xennet_move_rx_slot()
846 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; in xennet_move_rx_slot()
847 queue->rx.req_prod_pvt++; in xennet_move_rx_slot()
850 static int xennet_get_extras(struct netfront_queue *queue, in xennet_get_extras() argument
856 struct device *dev = &queue->info->netdev->dev; in xennet_get_extras()
857 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_extras()
867 err = -EBADR; in xennet_get_extras()
871 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); in xennet_get_extras()
878 err = -EINVAL; in xennet_get_extras()
880 extras[extra.type - 1] = extra; in xennet_get_extras()
883 skb = xennet_get_rx_skb(queue, cons); in xennet_get_extras()
884 ref = xennet_get_rx_ref(queue, cons); in xennet_get_extras()
885 xennet_move_rx_slot(queue, skb, ref); in xennet_get_extras()
888 queue->rx.rsp_cons = cons; in xennet_get_extras()
892 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, in xennet_run_xdp() argument
893 struct xen_netif_rx_response *rx, struct bpf_prog *prog, in xennet_run_xdp() argument
897 u32 len = rx->status; in xennet_run_xdp()
901 xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, in xennet_run_xdp()
902 &queue->xdp_rxq); in xennet_run_xdp()
911 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); in xennet_run_xdp()
915 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
919 err = xdp_do_redirect(queue->info->netdev, xdp, prog); in xennet_run_xdp()
922 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
929 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
939 static int xennet_get_responses(struct netfront_queue *queue, in xennet_get_responses() argument
944 struct xen_netif_rx_response *rx = &rinfo->rx, rx_local; in xennet_get_responses() local
945 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); in xennet_get_responses()
946 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_responses()
947 struct sk_buff *skb = xennet_get_rx_skb(queue, cons); in xennet_get_responses()
948 struct xen_netif_extra_info *extras = rinfo->extras; in xennet_get_responses()
949 grant_ref_t ref = xennet_get_rx_ref(queue, cons); in xennet_get_responses()
950 struct device *dev = &queue->info->netdev->dev; in xennet_get_responses()
958 if (rx->flags & XEN_NETRXF_extra_info) { in xennet_get_responses()
959 err = xennet_get_extras(queue, extras, rp); in xennet_get_responses()
961 if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) { in xennet_get_responses()
964 xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1]; in xennet_get_responses()
965 rx->offset = xdp->u.xdp.headroom; in xennet_get_responses()
968 cons = queue->rx.rsp_cons; in xennet_get_responses()
972 if (unlikely(rx->status < 0 || in xennet_get_responses()
973 rx->offset + rx->status > XEN_PAGE_SIZE)) { in xennet_get_responses()
975 dev_warn(dev, "rx->offset: %u, size: %d\n", in xennet_get_responses()
976 rx->offset, rx->status); in xennet_get_responses()
977 xennet_move_rx_slot(queue, skb, ref); in xennet_get_responses()
978 err = -EINVAL; in xennet_get_responses()
989 dev_warn(dev, "Bad rx response id %d.\n", in xennet_get_responses()
990 rx->id); in xennet_get_responses()
991 err = -EINVAL; in xennet_get_responses()
998 gnttab_release_grant_reference(&queue->gref_rx_head, ref); in xennet_get_responses()
1001 xdp_prog = rcu_dereference(queue->xdp_prog); in xennet_get_responses()
1003 if (!(rx->flags & XEN_NETRXF_more_data)) { in xennet_get_responses()
1005 verdict = xennet_run_xdp(queue, in xennet_get_responses()
1006 skb_frag_page(&skb_shinfo(skb)->frags[0]), in xennet_get_responses()
1007 rx, xdp_prog, &xdp, need_xdp_flush); in xennet_get_responses()
1009 err = -EINVAL; in xennet_get_responses()
1012 err = -EINVAL; in xennet_get_responses()
1018 if (!(rx->flags & XEN_NETRXF_more_data)) in xennet_get_responses()
1024 err = -ENOENT; in xennet_get_responses()
1028 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); in xennet_get_responses()
1029 rx = &rx_local; in xennet_get_responses()
1030 skb = xennet_get_rx_skb(queue, cons + slots); in xennet_get_responses()
1031 ref = xennet_get_rx_ref(queue, cons + slots); in xennet_get_responses()
1038 err = -E2BIG; in xennet_get_responses()
1042 queue->rx.rsp_cons = cons + slots; in xennet_get_responses()
1050 if (!gso->u.gso.size) { in xennet_set_skb_gso()
1053 return -EINVAL; in xennet_set_skb_gso()
1056 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && in xennet_set_skb_gso()
1057 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { in xennet_set_skb_gso()
1059 pr_warn("Bad GSO type %d\n", gso->u.gso.type); in xennet_set_skb_gso()
1060 return -EINVAL; in xennet_set_skb_gso()
1063 skb_shinfo(skb)->gso_size = gso->u.gso.size; in xennet_set_skb_gso()
1064 skb_shinfo(skb)->gso_type = in xennet_set_skb_gso()
1065 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? in xennet_set_skb_gso()
1070 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; in xennet_set_skb_gso()
1071 skb_shinfo(skb)->gso_segs = 0; in xennet_set_skb_gso()
1076 static int xennet_fill_frags(struct netfront_queue *queue, in xennet_fill_frags() argument
1080 RING_IDX cons = queue->rx.rsp_cons; in xennet_fill_frags()
1084 struct xen_netif_rx_response rx; in xennet_fill_frags() local
1085 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; in xennet_fill_frags()
1087 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); in xennet_fill_frags()
1089 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { in xennet_fill_frags()
1090 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in xennet_fill_frags()
1093 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in xennet_fill_frags()
1095 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { in xennet_fill_frags()
1096 queue->rx.rsp_cons = ++cons + skb_queue_len(list); in xennet_fill_frags()
1098 return -ENOENT; in xennet_fill_frags()
1101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in xennet_fill_frags()
1103 rx.offset, rx.status, PAGE_SIZE); in xennet_fill_frags()
1105 skb_shinfo(nskb)->nr_frags = 0; in xennet_fill_frags()
1109 queue->rx.rsp_cons = cons; in xennet_fill_frags()
1124 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { in checksum_setup()
1126 atomic_inc(&np->rx_gso_checksum_fixup); in checksum_setup()
1127 skb->ip_summed = CHECKSUM_PARTIAL; in checksum_setup()
1131 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ in checksum_setup()
1132 if (skb->ip_summed != CHECKSUM_PARTIAL) in checksum_setup()
1138 static int handle_incoming_queue(struct netfront_queue *queue, in handle_incoming_queue() argument
1141 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); in handle_incoming_queue()
1146 int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in handle_incoming_queue()
1149 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in handle_incoming_queue()
1152 skb->protocol = eth_type_trans(skb, queue->info->netdev); in handle_incoming_queue()
1155 if (checksum_setup(queue->info->netdev, skb)) { in handle_incoming_queue()
1158 queue->info->netdev->stats.rx_errors++; in handle_incoming_queue()
1162 u64_stats_update_begin(&rx_stats->syncp); in handle_incoming_queue()
1163 rx_stats->packets++; in handle_incoming_queue()
1164 rx_stats->bytes += skb->len; in handle_incoming_queue()
1165 u64_stats_update_end(&rx_stats->syncp); in handle_incoming_queue()
1168 napi_gro_receive(&queue->napi, skb); in handle_incoming_queue()
1176 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); in xennet_poll() local
1177 struct net_device *dev = queue->info->netdev; in xennet_poll()
1180 struct xen_netif_rx_response *rx = &rinfo.rx; in xennet_poll() local
1190 spin_lock(&queue->rx_lock); in xennet_poll()
1196 rp = queue->rx.sring->rsp_prod; in xennet_poll()
1197 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { in xennet_poll()
1198 dev_alert(&dev->dev, "Illegal number of responses %u\n", in xennet_poll()
1199 rp - queue->rx.rsp_cons); in xennet_poll()
1200 queue->info->broken = true; in xennet_poll()
1201 spin_unlock(&queue->rx_lock); in xennet_poll()
1206 i = queue->rx.rsp_cons; in xennet_poll()
1209 RING_COPY_RESPONSE(&queue->rx, i, rx); in xennet_poll()
1212 err = xennet_get_responses(queue, &rinfo, rp, &tmpq, in xennet_poll()
1219 dev->stats.rx_errors++; in xennet_poll()
1220 i = queue->rx.rsp_cons; in xennet_poll()
1226 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { in xennet_poll()
1228 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; in xennet_poll()
1232 queue->rx.rsp_cons += skb_queue_len(&tmpq); in xennet_poll()
1237 NETFRONT_SKB_CB(skb)->pull_to = rx->status; in xennet_poll()
1238 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) in xennet_poll()
1239 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; in xennet_poll()
1241 skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset); in xennet_poll()
1242 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); in xennet_poll()
1243 skb->data_len = rx->status; in xennet_poll()
1244 skb->len += rx->status; in xennet_poll()
1246 if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) in xennet_poll()
1249 if (rx->flags & XEN_NETRXF_csum_blank) in xennet_poll()
1250 skb->ip_summed = CHECKSUM_PARTIAL; in xennet_poll()
1251 else if (rx->flags & XEN_NETRXF_data_validated) in xennet_poll()
1252 skb->ip_summed = CHECKSUM_UNNECESSARY; in xennet_poll()
1256 i = ++queue->rx.rsp_cons; in xennet_poll()
1264 work_done -= handle_incoming_queue(queue, &rxq); in xennet_poll()
1266 xennet_alloc_rx_buffers(queue); in xennet_poll()
1273 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); in xennet_poll()
1278 spin_unlock(&queue->rx_lock); in xennet_poll()
1288 return -EINVAL; in xennet_change_mtu()
1289 dev->mtu = mtu; in xennet_change_mtu()
1300 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); in xennet_get_stats64()
1301 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); in xennet_get_stats64()
1306 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); in xennet_get_stats64()
1307 tx_packets = tx_stats->packets; in xennet_get_stats64()
1308 tx_bytes = tx_stats->bytes; in xennet_get_stats64()
1309 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); in xennet_get_stats64()
1312 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); in xennet_get_stats64()
1313 rx_packets = rx_stats->packets; in xennet_get_stats64()
1314 rx_bytes = rx_stats->bytes; in xennet_get_stats64()
1315 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); in xennet_get_stats64()
1317 tot->rx_packets += rx_packets; in xennet_get_stats64()
1318 tot->tx_packets += tx_packets; in xennet_get_stats64()
1319 tot->rx_bytes += rx_bytes; in xennet_get_stats64()
1320 tot->tx_bytes += tx_bytes; in xennet_get_stats64()
1323 tot->rx_errors = dev->stats.rx_errors; in xennet_get_stats64()
1324 tot->tx_dropped = dev->stats.tx_dropped; in xennet_get_stats64()
1327 static void xennet_release_tx_bufs(struct netfront_queue *queue) in xennet_release_tx_bufs() argument
1334 if (!queue->tx_skbs[i]) in xennet_release_tx_bufs()
1337 skb = queue->tx_skbs[i]; in xennet_release_tx_bufs()
1338 queue->tx_skbs[i] = NULL; in xennet_release_tx_bufs()
1339 get_page(queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1340 gnttab_end_foreign_access(queue->grant_tx_ref[i], in xennet_release_tx_bufs()
1342 (unsigned long)page_address(queue->grant_tx_page[i])); in xennet_release_tx_bufs()
1343 queue->grant_tx_page[i] = NULL; in xennet_release_tx_bufs()
1344 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_release_tx_bufs()
1345 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); in xennet_release_tx_bufs()
1350 static void xennet_release_rx_bufs(struct netfront_queue *queue) in xennet_release_rx_bufs() argument
1354 spin_lock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1360 skb = queue->rx_skbs[id]; in xennet_release_rx_bufs()
1364 ref = queue->grant_rx_ref[id]; in xennet_release_rx_bufs()
1368 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_release_rx_bufs()
1376 queue->grant_rx_ref[id] = GRANT_INVALID_REF; in xennet_release_rx_bufs()
1381 spin_unlock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1390 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) in xennet_fix_features()
1394 !xenbus_read_unsigned(np->xbdev->otherend, in xennet_fix_features()
1395 "feature-ipv6-csum-offload", 0)) in xennet_fix_features()
1399 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) in xennet_fix_features()
1403 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) in xennet_fix_features()
1412 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { in xennet_set_features()
1414 dev->mtu = ETH_DATA_LEN; in xennet_set_features()
1422 struct netfront_queue *queue = dev_id; in xennet_tx_interrupt() local
1425 if (queue->info->broken) in xennet_tx_interrupt()
1428 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_tx_interrupt()
1429 xennet_tx_buf_gc(queue); in xennet_tx_interrupt()
1430 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_tx_interrupt()
1437 struct netfront_queue *queue = dev_id; in xennet_rx_interrupt() local
1438 struct net_device *dev = queue->info->netdev; in xennet_rx_interrupt()
1440 if (queue->info->broken) in xennet_rx_interrupt()
1444 RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))) in xennet_rx_interrupt()
1445 napi_schedule(&queue->napi); in xennet_rx_interrupt()
1460 /* Poll each queue */ in xennet_poll_controller()
1462 unsigned int num_queues = dev->real_num_tx_queues; in xennet_poll_controller()
1465 if (info->broken) in xennet_poll_controller()
1469 xennet_interrupt(0, &info->queues[i]); in xennet_poll_controller()
1482 err = xenbus_printf(XBT_NIL, np->xbdev->nodename, in talk_to_netback_xdp()
1483 "xdp-headroom", "%hu", in talk_to_netback_xdp()
1486 pr_warn("Error writing xdp-headroom\n"); in talk_to_netback_xdp()
1494 unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; in xennet_xdp_set()
1499 if (dev->mtu > max_mtu) { in xennet_xdp_set()
1501 return -EINVAL; in xennet_xdp_set()
1504 if (!np->netback_has_xdp_headroom) in xennet_xdp_set()
1507 xenbus_switch_state(np->xbdev, XenbusStateReconfiguring); in xennet_xdp_set()
1516 xenbus_read_driver_state(np->xbdev->otherend) == in xennet_xdp_set()
1518 np->netfront_xdp_enabled = true; in xennet_xdp_set()
1520 old_prog = rtnl_dereference(np->queues[0].xdp_prog); in xennet_xdp_set()
1523 bpf_prog_add(prog, dev->real_num_tx_queues); in xennet_xdp_set()
1525 for (i = 0; i < dev->real_num_tx_queues; ++i) in xennet_xdp_set()
1526 rcu_assign_pointer(np->queues[i].xdp_prog, prog); in xennet_xdp_set()
1529 for (i = 0; i < dev->real_num_tx_queues; ++i) in xennet_xdp_set()
1532 xenbus_switch_state(np->xbdev, XenbusStateConnected); in xennet_xdp_set()
1541 if (np->broken) in xennet_xdp()
1542 return -ENODEV; in xennet_xdp()
1544 switch (xdp->command) { in xennet_xdp()
1546 return xennet_xdp_set(dev, xdp->prog, xdp->extack); in xennet_xdp()
1548 return -EINVAL; in xennet_xdp()
1574 free_percpu(np->rx_stats); in xennet_free_netdev()
1575 free_percpu(np->tx_stats); in xennet_free_netdev()
1587 return ERR_PTR(-ENOMEM); in xennet_create_dev()
1590 np->xbdev = dev; in xennet_create_dev()
1592 np->queues = NULL; in xennet_create_dev()
1594 err = -ENOMEM; in xennet_create_dev()
1595 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1596 if (np->rx_stats == NULL) in xennet_create_dev()
1598 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1599 if (np->tx_stats == NULL) in xennet_create_dev()
1602 netdev->netdev_ops = &xennet_netdev_ops; in xennet_create_dev()
1604 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | in xennet_create_dev()
1606 netdev->hw_features = NETIF_F_SG | in xennet_create_dev()
1616 netdev->features |= netdev->hw_features; in xennet_create_dev()
1618 netdev->ethtool_ops = &xennet_ethtool_ops; in xennet_create_dev()
1619 netdev->min_mtu = ETH_MIN_MTU; in xennet_create_dev()
1620 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; in xennet_create_dev()
1621 SET_NETDEV_DEV(netdev, &dev->dev); in xennet_create_dev()
1623 np->netdev = netdev; in xennet_create_dev()
1624 np->netfront_xdp_enabled = false; in xennet_create_dev()
1631 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1633 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1664 dev_set_drvdata(&dev->dev, info); in netfront_probe()
1666 info->netdev->sysfs_groups[0] = &xennet_dev_group; in netfront_probe()
1674 /* This frees the page as a side-effect */ in xennet_end_access()
1682 unsigned int num_queues = info->netdev->real_num_tx_queues; in xennet_disconnect_backend()
1684 netif_carrier_off(info->netdev); in xennet_disconnect_backend()
1686 for (i = 0; i < num_queues && info->queues; ++i) { in xennet_disconnect_backend()
1687 struct netfront_queue *queue = &info->queues[i]; in xennet_disconnect_backend() local
1689 del_timer_sync(&queue->rx_refill_timer); in xennet_disconnect_backend()
1691 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) in xennet_disconnect_backend()
1692 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1693 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { in xennet_disconnect_backend()
1694 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1695 unbind_from_irqhandler(queue->rx_irq, queue); in xennet_disconnect_backend()
1697 queue->tx_evtchn = queue->rx_evtchn = 0; in xennet_disconnect_backend()
1698 queue->tx_irq = queue->rx_irq = 0; in xennet_disconnect_backend()
1700 if (netif_running(info->netdev)) in xennet_disconnect_backend()
1701 napi_synchronize(&queue->napi); in xennet_disconnect_backend()
1703 xennet_release_tx_bufs(queue); in xennet_disconnect_backend()
1704 xennet_release_rx_bufs(queue); in xennet_disconnect_backend()
1705 gnttab_free_grant_references(queue->gref_tx_head); in xennet_disconnect_backend()
1706 gnttab_free_grant_references(queue->gref_rx_head); in xennet_disconnect_backend()
1709 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); in xennet_disconnect_backend()
1710 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); in xennet_disconnect_backend()
1712 queue->tx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1713 queue->rx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1714 queue->tx.sring = NULL; in xennet_disconnect_backend()
1715 queue->rx.sring = NULL; in xennet_disconnect_backend()
1717 page_pool_destroy(queue->page_pool); in xennet_disconnect_backend()
1724 * leave the device-layer structures intact so that this is transparent to the
1729 struct netfront_info *info = dev_get_drvdata(&dev->dev); in netfront_resume()
1731 dev_dbg(&dev->dev, "%s\n", dev->nodename); in netfront_resume()
1733 netif_tx_lock_bh(info->netdev); in netfront_resume()
1734 netif_device_detach(info->netdev); in netfront_resume()
1735 netif_tx_unlock_bh(info->netdev); in netfront_resume()
1746 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); in xen_net_read_mac()
1752 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { in xen_net_read_mac()
1754 return -ENOENT; in xen_net_read_mac()
1763 static int setup_netfront_single(struct netfront_queue *queue) in setup_netfront_single() argument
1767 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_single()
1771 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_single()
1773 0, queue->info->netdev->name, queue); in setup_netfront_single()
1776 queue->rx_evtchn = queue->tx_evtchn; in setup_netfront_single()
1777 queue->rx_irq = queue->tx_irq = err; in setup_netfront_single()
1782 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_single()
1783 queue->tx_evtchn = 0; in setup_netfront_single()
1788 static int setup_netfront_split(struct netfront_queue *queue) in setup_netfront_split() argument
1792 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_split()
1795 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); in setup_netfront_split()
1799 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), in setup_netfront_split()
1800 "%s-tx", queue->name); in setup_netfront_split()
1801 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_split()
1803 0, queue->tx_irq_name, queue); in setup_netfront_split()
1806 queue->tx_irq = err; in setup_netfront_split()
1808 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), in setup_netfront_split()
1809 "%s-rx", queue->name); in setup_netfront_split()
1810 err = bind_evtchn_to_irqhandler(queue->rx_evtchn, in setup_netfront_split()
1812 0, queue->rx_irq_name, queue); in setup_netfront_split()
1815 queue->rx_irq = err; in setup_netfront_split()
1820 unbind_from_irqhandler(queue->tx_irq, queue); in setup_netfront_split()
1821 queue->tx_irq = 0; in setup_netfront_split()
1823 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); in setup_netfront_split()
1824 queue->rx_evtchn = 0; in setup_netfront_split()
1826 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_split()
1827 queue->tx_evtchn = 0; in setup_netfront_split()
1833 struct netfront_queue *queue, unsigned int feature_split_evtchn) in setup_netfront() argument
1840 queue->tx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1841 queue->rx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1842 queue->rx.sring = NULL; in setup_netfront()
1843 queue->tx.sring = NULL; in setup_netfront()
1847 err = -ENOMEM; in setup_netfront()
1852 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); in setup_netfront()
1857 queue->tx_ring_ref = gref; in setup_netfront()
1861 err = -ENOMEM; in setup_netfront()
1862 xenbus_dev_fatal(dev, err, "allocating rx ring page"); in setup_netfront()
1866 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); in setup_netfront()
1871 queue->rx_ring_ref = gref; in setup_netfront()
1874 err = setup_netfront_split(queue); in setup_netfront()
1876 * a) feature-split-event-channels == 0 in setup_netfront()
1877 * b) feature-split-event-channels == 1 but failed to setup in setup_netfront()
1880 err = setup_netfront_single(queue); in setup_netfront()
1891 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); in setup_netfront()
1895 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); in setup_netfront()
1902 /* Queue-specific initialisation
1904 * be run per-queue.
1906 static int xennet_init_queue(struct netfront_queue *queue) in xennet_init_queue() argument
1912 spin_lock_init(&queue->tx_lock); in xennet_init_queue()
1913 spin_lock_init(&queue->rx_lock); in xennet_init_queue()
1915 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); in xennet_init_queue()
1917 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; in xennet_init_queue()
1918 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", in xennet_init_queue()
1919 devid, queue->id); in xennet_init_queue()
1922 queue->tx_skb_freelist = 0; in xennet_init_queue()
1923 queue->tx_pend_queue = TX_LINK_NONE; in xennet_init_queue()
1925 queue->tx_link[i] = i + 1; in xennet_init_queue()
1926 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1927 queue->grant_tx_page[i] = NULL; in xennet_init_queue()
1929 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; in xennet_init_queue()
1933 queue->rx_skbs[i] = NULL; in xennet_init_queue()
1934 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1939 &queue->gref_tx_head) < 0) { in xennet_init_queue()
1941 err = -ENOMEM; in xennet_init_queue()
1945 /* A grant for every rx ring slot */ in xennet_init_queue()
1947 &queue->gref_rx_head) < 0) { in xennet_init_queue()
1948 pr_alert("can't alloc rx grant refs\n"); in xennet_init_queue()
1949 err = -ENOMEM; in xennet_init_queue()
1956 gnttab_free_grant_references(queue->gref_tx_head); in xennet_init_queue()
1961 static int write_queue_xenstore_keys(struct netfront_queue *queue, in write_queue_xenstore_keys() argument
1964 /* Write the queue-specific keys into XenStore in the traditional in write_queue_xenstore_keys()
1965 * way for a single queue, or in a queue subkeys for multiple in write_queue_xenstore_keys()
1968 struct xenbus_device *dev = queue->info->xbdev; in write_queue_xenstore_keys()
1976 pathsize = strlen(dev->nodename) + 10; in write_queue_xenstore_keys()
1979 err = -ENOMEM; in write_queue_xenstore_keys()
1983 snprintf(path, pathsize, "%s/queue-%u", in write_queue_xenstore_keys()
1984 dev->nodename, queue->id); in write_queue_xenstore_keys()
1986 path = (char *)dev->nodename; in write_queue_xenstore_keys()
1990 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", in write_queue_xenstore_keys()
1991 queue->tx_ring_ref); in write_queue_xenstore_keys()
1993 message = "writing tx-ring-ref"; in write_queue_xenstore_keys()
1997 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", in write_queue_xenstore_keys()
1998 queue->rx_ring_ref); in write_queue_xenstore_keys()
2000 message = "writing rx-ring-ref"; in write_queue_xenstore_keys()
2007 if (queue->tx_evtchn == queue->rx_evtchn) { in write_queue_xenstore_keys()
2010 "event-channel", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2012 message = "writing event-channel"; in write_queue_xenstore_keys()
2018 "event-channel-tx", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2020 message = "writing event-channel-tx"; in write_queue_xenstore_keys()
2025 "event-channel-rx", "%u", queue->rx_evtchn); in write_queue_xenstore_keys()
2027 message = "writing event-channel-rx"; in write_queue_xenstore_keys()
2047 for (i = 0; i < info->netdev->real_num_tx_queues; i++) { in xennet_destroy_queues()
2048 struct netfront_queue *queue = &info->queues[i]; in xennet_destroy_queues() local
2050 if (netif_running(info->netdev)) in xennet_destroy_queues()
2051 napi_disable(&queue->napi); in xennet_destroy_queues()
2052 netif_napi_del(&queue->napi); in xennet_destroy_queues()
2055 kfree(info->queues); in xennet_destroy_queues()
2056 info->queues = NULL; in xennet_destroy_queues()
2061 static int xennet_create_page_pool(struct netfront_queue *queue) in xennet_create_page_pool() argument
2069 .dev = &queue->info->netdev->dev, in xennet_create_page_pool()
2071 .max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, in xennet_create_page_pool()
2074 queue->page_pool = page_pool_create(&pp_params); in xennet_create_page_pool()
2075 if (IS_ERR(queue->page_pool)) { in xennet_create_page_pool()
2076 err = PTR_ERR(queue->page_pool); in xennet_create_page_pool()
2077 queue->page_pool = NULL; in xennet_create_page_pool()
2081 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, in xennet_create_page_pool()
2082 queue->id, 0); in xennet_create_page_pool()
2084 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); in xennet_create_page_pool()
2088 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, in xennet_create_page_pool()
2089 MEM_TYPE_PAGE_POOL, queue->page_pool); in xennet_create_page_pool()
2091 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); in xennet_create_page_pool()
2097 xdp_rxq_info_unreg(&queue->xdp_rxq); in xennet_create_page_pool()
2099 page_pool_destroy(queue->page_pool); in xennet_create_page_pool()
2100 queue->page_pool = NULL; in xennet_create_page_pool()
2110 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), in xennet_create_queues()
2112 if (!info->queues) in xennet_create_queues()
2113 return -ENOMEM; in xennet_create_queues()
2116 struct netfront_queue *queue = &info->queues[i]; in xennet_create_queues() local
2118 queue->id = i; in xennet_create_queues()
2119 queue->info = info; in xennet_create_queues()
2121 ret = xennet_init_queue(queue); in xennet_create_queues()
2123 dev_warn(&info->xbdev->dev, in xennet_create_queues()
2130 ret = xennet_create_page_pool(queue); in xennet_create_queues()
2132 dev_err(&info->xbdev->dev, "can't allocate page pool\n"); in xennet_create_queues()
2137 netif_napi_add(queue->info->netdev, &queue->napi, in xennet_create_queues()
2139 if (netif_running(info->netdev)) in xennet_create_queues()
2140 napi_enable(&queue->napi); in xennet_create_queues()
2143 netif_set_real_num_tx_queues(info->netdev, *num_queues); in xennet_create_queues()
2146 dev_err(&info->xbdev->dev, "no queues\n"); in xennet_create_queues()
2147 return -EINVAL; in xennet_create_queues()
2162 struct netfront_queue *queue = NULL; in talk_to_netback() local
2165 info->netdev->irq = 0; in talk_to_netback()
2168 max_queues = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2169 "multi-queue-max-queues", 1); in talk_to_netback()
2172 /* Check feature-split-event-channels */ in talk_to_netback()
2173 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2174 "feature-split-event-channels", 0); in talk_to_netback()
2177 err = xen_net_read_mac(dev, info->netdev->dev_addr); in talk_to_netback()
2179 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); in talk_to_netback()
2183 info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2184 "feature-xdp-headroom", 0); in talk_to_netback()
2185 if (info->netback_has_xdp_headroom) { in talk_to_netback()
2186 /* set the current xen-netfront xdp state */ in talk_to_netback()
2187 err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ? in talk_to_netback()
2195 if (info->queues) in talk_to_netback()
2199 info->broken = false; in talk_to_netback()
2204 kfree(info->queues); in talk_to_netback()
2205 info->queues = NULL; in talk_to_netback()
2210 /* Create shared ring, alloc event channel -- for each queue */ in talk_to_netback()
2212 queue = &info->queues[i]; in talk_to_netback()
2213 err = setup_netfront(dev, queue, feature_split_evtchn); in talk_to_netback()
2226 info->xbdev->otherend, "multi-queue-max-queues")) { in talk_to_netback()
2228 err = xenbus_printf(xbt, dev->nodename, in talk_to_netback()
2229 "multi-queue-num-queues", "%u", num_queues); in talk_to_netback()
2231 message = "writing multi-queue-num-queues"; in talk_to_netback()
2237 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ in talk_to_netback()
2241 /* Write the keys for each queue */ in talk_to_netback()
2243 queue = &info->queues[i]; in talk_to_netback()
2244 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ in talk_to_netback()
2250 /* The remaining keys are not queue-specific */ in talk_to_netback()
2251 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", in talk_to_netback()
2254 message = "writing request-rx-copy"; in talk_to_netback()
2258 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); in talk_to_netback()
2260 message = "writing feature-rx-notify"; in talk_to_netback()
2264 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); in talk_to_netback()
2266 message = "writing feature-sg"; in talk_to_netback()
2270 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); in talk_to_netback()
2272 message = "writing feature-gso-tcpv4"; in talk_to_netback()
2276 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); in talk_to_netback()
2278 message = "writing feature-gso-tcpv6"; in talk_to_netback()
2282 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", in talk_to_netback()
2285 message = "writing feature-ipv6-csum-offload"; in talk_to_netback()
2291 if (err == -EAGAIN) in talk_to_netback()
2310 device_unregister(&dev->dev); in talk_to_netback()
2320 struct netfront_queue *queue = NULL; in xennet_connect() local
2322 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { in xennet_connect()
2323 dev_info(&dev->dev, in xennet_connect()
2325 return -ENODEV; in xennet_connect()
2328 err = talk_to_netback(np->xbdev, np); in xennet_connect()
2331 if (np->netback_has_xdp_headroom) in xennet_connect()
2335 num_queues = dev->real_num_tx_queues; in xennet_connect()
2337 if (dev->reg_state == NETREG_UNINITIALIZED) { in xennet_connect()
2341 device_unregister(&np->xbdev->dev); in xennet_connect()
2356 netif_tx_lock_bh(np->netdev); in xennet_connect()
2357 netif_device_attach(np->netdev); in xennet_connect()
2358 netif_tx_unlock_bh(np->netdev); in xennet_connect()
2360 netif_carrier_on(np->netdev); in xennet_connect()
2362 queue = &np->queues[j]; in xennet_connect()
2364 notify_remote_via_irq(queue->tx_irq); in xennet_connect()
2365 if (queue->tx_irq != queue->rx_irq) in xennet_connect()
2366 notify_remote_via_irq(queue->rx_irq); in xennet_connect()
2368 spin_lock_irq(&queue->tx_lock); in xennet_connect()
2369 xennet_tx_buf_gc(queue); in xennet_connect()
2370 spin_unlock_irq(&queue->tx_lock); in xennet_connect()
2372 spin_lock_bh(&queue->rx_lock); in xennet_connect()
2373 xennet_alloc_rx_buffers(queue); in xennet_connect()
2374 spin_unlock_bh(&queue->rx_lock); in xennet_connect()
2386 struct netfront_info *np = dev_get_drvdata(&dev->dev); in netback_changed()
2387 struct net_device *netdev = np->netdev; in netback_changed()
2389 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); in netback_changed()
2402 if (dev->state != XenbusStateInitialising) in netback_changed()
2414 if (dev->state == XenbusStateClosed) in netback_changed()
2439 return -EINVAL; in xennet_get_sset_count()
2490 return -EPERM; in store_rxbuf()
2494 return -EBADMSG; in store_rxbuf()
2521 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2526 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2528 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2530 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2535 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2541 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2543 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2551 struct netfront_info *info = dev_get_drvdata(&dev->dev); in xennet_remove()
2556 if (info->netdev->reg_state == NETREG_REGISTERED) in xennet_remove()
2557 unregister_netdev(info->netdev); in xennet_remove()
2559 if (info->queues) { in xennet_remove()
2564 xennet_free_netdev(info->netdev); in xennet_remove()
2585 return -ENODEV; in netif_init()
2588 return -ENODEV; in netif_init()