Lines Matching full:queue
89 /* Queue name is interface name with "-qNNN" appended */
92 /* IRQ name is queue name with "-tx" or "-rx" appended */
106 unsigned int id; /* Queue ID, 0-based */
165 /* Multi-queue support */
219 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, in xennet_get_rx_skb() argument
223 struct sk_buff *skb = queue->rx_skbs[i]; in xennet_get_rx_skb()
224 queue->rx_skbs[i] = NULL; in xennet_get_rx_skb()
228 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, in xennet_get_rx_ref() argument
232 grant_ref_t ref = queue->grant_rx_ref[i]; in xennet_get_rx_ref()
233 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_get_rx_ref()
249 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); in rx_refill_timeout() local
250 napi_schedule(&queue->napi); in rx_refill_timeout()
253 static int netfront_tx_slot_available(struct netfront_queue *queue) in netfront_tx_slot_available() argument
255 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < in netfront_tx_slot_available()
259 static void xennet_maybe_wake_tx(struct netfront_queue *queue) in xennet_maybe_wake_tx() argument
261 struct net_device *dev = queue->info->netdev; in xennet_maybe_wake_tx()
262 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); in xennet_maybe_wake_tx()
265 netfront_tx_slot_available(queue) && in xennet_maybe_wake_tx()
267 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_maybe_wake_tx()
271 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) in xennet_alloc_one_rx_buffer() argument
276 skb = __netdev_alloc_skb(queue->info->netdev, in xennet_alloc_one_rx_buffer()
282 page = page_pool_dev_alloc_pages(queue->page_pool); in xennet_alloc_one_rx_buffer()
291 skb->dev = queue->info->netdev; in xennet_alloc_one_rx_buffer()
297 static void xennet_alloc_rx_buffers(struct netfront_queue *queue) in xennet_alloc_rx_buffers() argument
299 RING_IDX req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
303 if (unlikely(!netif_carrier_ok(queue->info->netdev))) in xennet_alloc_rx_buffers()
306 for (req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
307 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; in xennet_alloc_rx_buffers()
315 skb = xennet_alloc_one_rx_buffer(queue); in xennet_alloc_rx_buffers()
323 BUG_ON(queue->rx_skbs[id]); in xennet_alloc_rx_buffers()
324 queue->rx_skbs[id] = skb; in xennet_alloc_rx_buffers()
326 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); in xennet_alloc_rx_buffers()
328 queue->grant_rx_ref[id] = ref; in xennet_alloc_rx_buffers()
332 req = RING_GET_REQUEST(&queue->rx, req_prod); in xennet_alloc_rx_buffers()
334 queue->info->xbdev->otherend_id, in xennet_alloc_rx_buffers()
341 queue->rx.req_prod_pvt = req_prod; in xennet_alloc_rx_buffers()
348 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || in xennet_alloc_rx_buffers()
350 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); in xennet_alloc_rx_buffers()
354 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); in xennet_alloc_rx_buffers()
356 notify_remote_via_irq(queue->rx_irq); in xennet_alloc_rx_buffers()
364 struct netfront_queue *queue = NULL; in xennet_open() local
370 queue = &np->queues[i]; in xennet_open()
371 napi_enable(&queue->napi); in xennet_open()
373 spin_lock_bh(&queue->rx_lock); in xennet_open()
375 xennet_alloc_rx_buffers(queue); in xennet_open()
376 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; in xennet_open()
377 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) in xennet_open()
378 napi_schedule(&queue->napi); in xennet_open()
380 spin_unlock_bh(&queue->rx_lock); in xennet_open()
388 static void xennet_tx_buf_gc(struct netfront_queue *queue) in xennet_tx_buf_gc() argument
395 BUG_ON(!netif_carrier_ok(queue->info->netdev)); in xennet_tx_buf_gc()
398 prod = queue->tx.sring->rsp_prod; in xennet_tx_buf_gc()
401 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { in xennet_tx_buf_gc()
404 txrsp = RING_GET_RESPONSE(&queue->tx, cons); in xennet_tx_buf_gc()
409 skb = queue->tx_skbs[id].skb; in xennet_tx_buf_gc()
411 queue->grant_tx_ref[id]) != 0)) { in xennet_tx_buf_gc()
417 queue->grant_tx_ref[id], GNTMAP_readonly); in xennet_tx_buf_gc()
419 &queue->gref_tx_head, queue->grant_tx_ref[id]); in xennet_tx_buf_gc()
420 queue->grant_tx_ref[id] = GRANT_INVALID_REF; in xennet_tx_buf_gc()
421 queue->grant_tx_page[id] = NULL; in xennet_tx_buf_gc()
422 add_id_to_freelist(&queue->tx_skb_freelist, queue->tx_skbs, id); in xennet_tx_buf_gc()
426 queue->tx.rsp_cons = prod; in xennet_tx_buf_gc()
428 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); in xennet_tx_buf_gc()
431 xennet_maybe_wake_tx(queue); in xennet_tx_buf_gc()
435 struct netfront_queue *queue; member
451 struct netfront_queue *queue = info->queue; in xennet_tx_setup_grant() local
454 id = get_id_from_freelist(&queue->tx_skb_freelist, queue->tx_skbs); in xennet_tx_setup_grant()
455 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_tx_setup_grant()
456 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); in xennet_tx_setup_grant()
459 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, in xennet_tx_setup_grant()
462 queue->tx_skbs[id].skb = skb; in xennet_tx_setup_grant()
463 queue->grant_tx_page[id] = page; in xennet_tx_setup_grant()
464 queue->grant_tx_ref[id] = ref; in xennet_tx_setup_grant()
477 struct netfront_queue *queue, struct sk_buff *skb, in xennet_make_first_txreq() argument
481 .queue = queue, in xennet_make_first_txreq()
503 struct netfront_queue *queue, struct xen_netif_tx_request *tx, in xennet_make_txreqs() argument
508 .queue = queue, in xennet_make_txreqs()
566 /* First, check if there is only one queue */ in xennet_select_queue()
578 struct netfront_queue *queue, in xennet_xdp_xmit_one() argument
585 xennet_make_first_txreq(queue, NULL, in xennet_xdp_xmit_one()
590 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_xdp_xmit_one()
592 notify_remote_via_irq(queue->tx_irq); in xennet_xdp_xmit_one()
599 xennet_tx_buf_gc(queue); in xennet_xdp_xmit_one()
609 struct netfront_queue *queue = NULL; in xennet_xdp_xmit() local
617 queue = &np->queues[smp_processor_id() % num_queues]; in xennet_xdp_xmit()
619 spin_lock_irqsave(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
625 err = xennet_xdp_xmit_one(dev, queue, xdpf); in xennet_xdp_xmit()
631 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
651 struct netfront_queue *queue = NULL; in xennet_start_xmit() local
659 /* Determine which queue to transmit this SKB on */ in xennet_start_xmit()
661 queue = &np->queues[queue_index]; in xennet_start_xmit()
699 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_start_xmit()
704 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
709 first_tx = tx = xennet_make_first_txreq(queue, skb, in xennet_start_xmit()
730 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_start_xmit()
746 tx = xennet_make_txreqs(queue, tx, skb, page, offset, len); in xennet_start_xmit()
751 tx = xennet_make_txreqs(queue, tx, skb, skb_frag_page(frag), in xennet_start_xmit()
762 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_start_xmit()
764 notify_remote_via_irq(queue->tx_irq); in xennet_start_xmit()
772 xennet_tx_buf_gc(queue); in xennet_start_xmit()
774 if (!netfront_tx_slot_available(queue)) in xennet_start_xmit()
775 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_start_xmit()
777 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
792 struct netfront_queue *queue; in xennet_close() local
795 queue = &np->queues[i]; in xennet_close()
796 napi_disable(&queue->napi); in xennet_close()
801 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, in xennet_move_rx_slot() argument
804 int new = xennet_rxidx(queue->rx.req_prod_pvt); in xennet_move_rx_slot()
806 BUG_ON(queue->rx_skbs[new]); in xennet_move_rx_slot()
807 queue->rx_skbs[new] = skb; in xennet_move_rx_slot()
808 queue->grant_rx_ref[new] = ref; in xennet_move_rx_slot()
809 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; in xennet_move_rx_slot()
810 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; in xennet_move_rx_slot()
811 queue->rx.req_prod_pvt++; in xennet_move_rx_slot()
814 static int xennet_get_extras(struct netfront_queue *queue, in xennet_get_extras() argument
820 struct device *dev = &queue->info->netdev->dev; in xennet_get_extras()
821 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_extras()
836 RING_GET_RESPONSE(&queue->rx, ++cons); in xennet_get_extras()
849 skb = xennet_get_rx_skb(queue, cons); in xennet_get_extras()
850 ref = xennet_get_rx_ref(queue, cons); in xennet_get_extras()
851 xennet_move_rx_slot(queue, skb, ref); in xennet_get_extras()
854 queue->rx.rsp_cons = cons; in xennet_get_extras()
858 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, in xennet_run_xdp() argument
871 xdp->rxq = &queue->xdp_rxq; in xennet_run_xdp()
879 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); in xennet_run_xdp()
881 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
885 err = xdp_do_redirect(queue->info->netdev, xdp, prog); in xennet_run_xdp()
888 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
895 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
905 static int xennet_get_responses(struct netfront_queue *queue, in xennet_get_responses() argument
912 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_responses()
913 struct sk_buff *skb = xennet_get_rx_skb(queue, cons); in xennet_get_responses()
915 grant_ref_t ref = xennet_get_rx_ref(queue, cons); in xennet_get_responses()
916 struct device *dev = &queue->info->netdev->dev; in xennet_get_responses()
925 err = xennet_get_extras(queue, extras, rp); in xennet_get_responses()
934 cons = queue->rx.rsp_cons; in xennet_get_responses()
943 xennet_move_rx_slot(queue, skb, ref); in xennet_get_responses()
964 gnttab_release_grant_reference(&queue->gref_rx_head, ref); in xennet_get_responses()
967 xdp_prog = rcu_dereference(queue->xdp_prog); in xennet_get_responses()
971 verdict = xennet_run_xdp(queue, in xennet_get_responses()
994 rx = RING_GET_RESPONSE(&queue->rx, cons + slots); in xennet_get_responses()
995 skb = xennet_get_rx_skb(queue, cons + slots); in xennet_get_responses()
996 ref = xennet_get_rx_ref(queue, cons + slots); in xennet_get_responses()
1007 queue->rx.rsp_cons = cons + slots; in xennet_get_responses()
1041 static int xennet_fill_frags(struct netfront_queue *queue, in xennet_fill_frags() argument
1045 RING_IDX cons = queue->rx.rsp_cons; in xennet_fill_frags()
1050 RING_GET_RESPONSE(&queue->rx, ++cons); in xennet_fill_frags()
1060 queue->rx.rsp_cons = ++cons + skb_queue_len(list); in xennet_fill_frags()
1073 queue->rx.rsp_cons = cons; in xennet_fill_frags()
1102 static int handle_incoming_queue(struct netfront_queue *queue, in handle_incoming_queue() argument
1105 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); in handle_incoming_queue()
1116 skb->protocol = eth_type_trans(skb, queue->info->netdev); in handle_incoming_queue()
1119 if (checksum_setup(queue->info->netdev, skb)) { in handle_incoming_queue()
1122 queue->info->netdev->stats.rx_errors++; in handle_incoming_queue()
1132 napi_gro_receive(&queue->napi, skb); in handle_incoming_queue()
1140 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); in xennet_poll() local
1141 struct net_device *dev = queue->info->netdev; in xennet_poll()
1154 spin_lock(&queue->rx_lock); in xennet_poll()
1160 rp = queue->rx.sring->rsp_prod; in xennet_poll()
1163 i = queue->rx.rsp_cons; in xennet_poll()
1166 memcpy(rx, RING_GET_RESPONSE(&queue->rx, i), sizeof(*rx)); in xennet_poll()
1169 err = xennet_get_responses(queue, &rinfo, rp, &tmpq, in xennet_poll()
1177 i = queue->rx.rsp_cons; in xennet_poll()
1189 queue->rx.rsp_cons += skb_queue_len(&tmpq); in xennet_poll()
1203 if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) in xennet_poll()
1213 i = ++queue->rx.rsp_cons; in xennet_poll()
1221 work_done -= handle_incoming_queue(queue, &rxq); in xennet_poll()
1223 xennet_alloc_rx_buffers(queue); in xennet_poll()
1230 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); in xennet_poll()
1235 spin_unlock(&queue->rx_lock); in xennet_poll()
1284 static void xennet_release_tx_bufs(struct netfront_queue *queue) in xennet_release_tx_bufs() argument
1291 if (skb_entry_is_link(&queue->tx_skbs[i])) in xennet_release_tx_bufs()
1294 skb = queue->tx_skbs[i].skb; in xennet_release_tx_bufs()
1295 get_page(queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1296 gnttab_end_foreign_access(queue->grant_tx_ref[i], in xennet_release_tx_bufs()
1298 (unsigned long)page_address(queue->grant_tx_page[i])); in xennet_release_tx_bufs()
1299 queue->grant_tx_page[i] = NULL; in xennet_release_tx_bufs()
1300 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_release_tx_bufs()
1301 add_id_to_freelist(&queue->tx_skb_freelist, queue->tx_skbs, i); in xennet_release_tx_bufs()
1306 static void xennet_release_rx_bufs(struct netfront_queue *queue) in xennet_release_rx_bufs() argument
1310 spin_lock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1316 skb = queue->rx_skbs[id]; in xennet_release_rx_bufs()
1320 ref = queue->grant_rx_ref[id]; in xennet_release_rx_bufs()
1332 queue->grant_rx_ref[id] = GRANT_INVALID_REF; in xennet_release_rx_bufs()
1337 spin_unlock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1378 struct netfront_queue *queue = dev_id; in xennet_tx_interrupt() local
1381 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_tx_interrupt()
1382 xennet_tx_buf_gc(queue); in xennet_tx_interrupt()
1383 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_tx_interrupt()
1390 struct netfront_queue *queue = dev_id; in xennet_rx_interrupt() local
1391 struct net_device *dev = queue->info->netdev; in xennet_rx_interrupt()
1394 RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))) in xennet_rx_interrupt()
1395 napi_schedule(&queue->napi); in xennet_rx_interrupt()
1410 /* Poll each queue */ in xennet_poll_controller()
1628 struct netfront_queue *queue = &info->queues[i]; in xennet_disconnect_backend() local
1630 del_timer_sync(&queue->rx_refill_timer); in xennet_disconnect_backend()
1632 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) in xennet_disconnect_backend()
1633 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1634 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { in xennet_disconnect_backend()
1635 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1636 unbind_from_irqhandler(queue->rx_irq, queue); in xennet_disconnect_backend()
1638 queue->tx_evtchn = queue->rx_evtchn = 0; in xennet_disconnect_backend()
1639 queue->tx_irq = queue->rx_irq = 0; in xennet_disconnect_backend()
1642 napi_synchronize(&queue->napi); in xennet_disconnect_backend()
1644 xennet_release_tx_bufs(queue); in xennet_disconnect_backend()
1645 xennet_release_rx_bufs(queue); in xennet_disconnect_backend()
1646 gnttab_free_grant_references(queue->gref_tx_head); in xennet_disconnect_backend()
1647 gnttab_free_grant_references(queue->gref_rx_head); in xennet_disconnect_backend()
1650 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); in xennet_disconnect_backend()
1651 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); in xennet_disconnect_backend()
1653 queue->tx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1654 queue->rx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1655 queue->tx.sring = NULL; in xennet_disconnect_backend()
1656 queue->rx.sring = NULL; in xennet_disconnect_backend()
1658 page_pool_destroy(queue->page_pool); in xennet_disconnect_backend()
1700 static int setup_netfront_single(struct netfront_queue *queue) in setup_netfront_single() argument
1704 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_single()
1708 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_single()
1710 0, queue->info->netdev->name, queue); in setup_netfront_single()
1713 queue->rx_evtchn = queue->tx_evtchn; in setup_netfront_single()
1714 queue->rx_irq = queue->tx_irq = err; in setup_netfront_single()
1719 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_single()
1720 queue->tx_evtchn = 0; in setup_netfront_single()
1725 static int setup_netfront_split(struct netfront_queue *queue) in setup_netfront_split() argument
1729 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_split()
1732 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); in setup_netfront_split()
1736 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), in setup_netfront_split()
1737 "%s-tx", queue->name); in setup_netfront_split()
1738 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_split()
1740 0, queue->tx_irq_name, queue); in setup_netfront_split()
1743 queue->tx_irq = err; in setup_netfront_split()
1745 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), in setup_netfront_split()
1746 "%s-rx", queue->name); in setup_netfront_split()
1747 err = bind_evtchn_to_irqhandler(queue->rx_evtchn, in setup_netfront_split()
1749 0, queue->rx_irq_name, queue); in setup_netfront_split()
1752 queue->rx_irq = err; in setup_netfront_split()
1757 unbind_from_irqhandler(queue->tx_irq, queue); in setup_netfront_split()
1758 queue->tx_irq = 0; in setup_netfront_split()
1760 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); in setup_netfront_split()
1761 queue->rx_evtchn = 0; in setup_netfront_split()
1763 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_split()
1764 queue->tx_evtchn = 0; in setup_netfront_split()
1770 struct netfront_queue *queue, unsigned int feature_split_evtchn) in setup_netfront() argument
1777 queue->tx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1778 queue->rx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1779 queue->rx.sring = NULL; in setup_netfront()
1780 queue->tx.sring = NULL; in setup_netfront()
1789 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); in setup_netfront()
1794 queue->tx_ring_ref = gref; in setup_netfront()
1803 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); in setup_netfront()
1808 queue->rx_ring_ref = gref; in setup_netfront()
1811 err = setup_netfront_split(queue); in setup_netfront()
1817 err = setup_netfront_single(queue); in setup_netfront()
1828 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); in setup_netfront()
1832 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); in setup_netfront()
1839 /* Queue-specific initialisation
1841 * be run per-queue.
1843 static int xennet_init_queue(struct netfront_queue *queue) in xennet_init_queue() argument
1849 spin_lock_init(&queue->tx_lock); in xennet_init_queue()
1850 spin_lock_init(&queue->rx_lock); in xennet_init_queue()
1852 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); in xennet_init_queue()
1854 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; in xennet_init_queue()
1855 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", in xennet_init_queue()
1856 devid, queue->id); in xennet_init_queue()
1859 queue->tx_skb_freelist = 0; in xennet_init_queue()
1861 skb_entry_set_link(&queue->tx_skbs[i], i+1); in xennet_init_queue()
1862 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1863 queue->grant_tx_page[i] = NULL; in xennet_init_queue()
1868 queue->rx_skbs[i] = NULL; in xennet_init_queue()
1869 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1874 &queue->gref_tx_head) < 0) { in xennet_init_queue()
1882 &queue->gref_rx_head) < 0) { in xennet_init_queue()
1891 gnttab_free_grant_references(queue->gref_tx_head); in xennet_init_queue()
1896 static int write_queue_xenstore_keys(struct netfront_queue *queue, in write_queue_xenstore_keys() argument
1899 /* Write the queue-specific keys into XenStore in the traditional in write_queue_xenstore_keys()
1900 * way for a single queue, or in a queue subkeys for multiple in write_queue_xenstore_keys()
1903 struct xenbus_device *dev = queue->info->xbdev; in write_queue_xenstore_keys()
1918 snprintf(path, pathsize, "%s/queue-%u", in write_queue_xenstore_keys()
1919 dev->nodename, queue->id); in write_queue_xenstore_keys()
1926 queue->tx_ring_ref); in write_queue_xenstore_keys()
1933 queue->rx_ring_ref); in write_queue_xenstore_keys()
1942 if (queue->tx_evtchn == queue->rx_evtchn) { in write_queue_xenstore_keys()
1945 "event-channel", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
1953 "event-channel-tx", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
1960 "event-channel-rx", "%u", queue->rx_evtchn); in write_queue_xenstore_keys()
1983 struct netfront_queue *queue = &info->queues[i]; in xennet_destroy_queues() local
1986 napi_disable(&queue->napi); in xennet_destroy_queues()
1987 netif_napi_del(&queue->napi); in xennet_destroy_queues()
1996 static int xennet_create_page_pool(struct netfront_queue *queue) in xennet_create_page_pool() argument
2004 .dev = &queue->info->netdev->dev, in xennet_create_page_pool()
2009 queue->page_pool = page_pool_create(&pp_params); in xennet_create_page_pool()
2010 if (IS_ERR(queue->page_pool)) { in xennet_create_page_pool()
2011 err = PTR_ERR(queue->page_pool); in xennet_create_page_pool()
2012 queue->page_pool = NULL; in xennet_create_page_pool()
2016 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, in xennet_create_page_pool()
2017 queue->id); in xennet_create_page_pool()
2019 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); in xennet_create_page_pool()
2023 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, in xennet_create_page_pool()
2024 MEM_TYPE_PAGE_POOL, queue->page_pool); in xennet_create_page_pool()
2026 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); in xennet_create_page_pool()
2032 xdp_rxq_info_unreg(&queue->xdp_rxq); in xennet_create_page_pool()
2034 page_pool_destroy(queue->page_pool); in xennet_create_page_pool()
2035 queue->page_pool = NULL; in xennet_create_page_pool()
2051 struct netfront_queue *queue = &info->queues[i]; in xennet_create_queues() local
2053 queue->id = i; in xennet_create_queues()
2054 queue->info = info; in xennet_create_queues()
2056 ret = xennet_init_queue(queue); in xennet_create_queues()
2065 ret = xennet_create_page_pool(queue); in xennet_create_queues()
2072 netif_napi_add(queue->info->netdev, &queue->napi, in xennet_create_queues()
2075 napi_enable(&queue->napi); in xennet_create_queues()
2097 struct netfront_queue *queue = NULL; in talk_to_netback() local
2104 "multi-queue-max-queues", 1); in talk_to_netback()
2142 /* Create shared ring, alloc event channel -- for each queue */ in talk_to_netback()
2144 queue = &info->queues[i]; in talk_to_netback()
2145 err = setup_netfront(dev, queue, feature_split_evtchn); in talk_to_netback()
2158 info->xbdev->otherend, "multi-queue-max-queues")) { in talk_to_netback()
2161 "multi-queue-num-queues", "%u", num_queues); in talk_to_netback()
2163 message = "writing multi-queue-num-queues"; in talk_to_netback()
2173 /* Write the keys for each queue */ in talk_to_netback()
2175 queue = &info->queues[i]; in talk_to_netback()
2176 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ in talk_to_netback()
2182 /* The remaining keys are not queue-specific */ in talk_to_netback()
2252 struct netfront_queue *queue = NULL; in xennet_connect() local
2290 queue = &np->queues[j]; in xennet_connect()
2292 notify_remote_via_irq(queue->tx_irq); in xennet_connect()
2293 if (queue->tx_irq != queue->rx_irq) in xennet_connect()
2294 notify_remote_via_irq(queue->rx_irq); in xennet_connect()
2296 spin_lock_irq(&queue->tx_lock); in xennet_connect()
2297 xennet_tx_buf_gc(queue); in xennet_connect()
2298 spin_unlock_irq(&queue->tx_lock); in xennet_connect()
2300 spin_lock_bh(&queue->rx_lock); in xennet_connect()
2301 xennet_alloc_rx_buffers(queue); in xennet_connect()
2302 spin_unlock_bh(&queue->rx_lock); in xennet_connect()