47 /* Number of bytes allowed on the internal guest Rx queue. */
55 void xenvif_skb_zerocopy_prepare(struct xenvif_queue *queue,  in xenvif_skb_zerocopy_prepare()  argument
59 	atomic_inc(&queue->inflight_packets);  in xenvif_skb_zerocopy_prepare()
62 void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue)  in xenvif_skb_zerocopy_complete()  argument
64 	atomic_dec(&queue->inflight_packets);  in xenvif_skb_zerocopy_complete()
70 	wake_up(&queue->dealloc_wq);  in xenvif_skb_zerocopy_complete()
80 static bool xenvif_handle_tx_interrupt(struct xenvif_queue *queue)  in xenvif_handle_tx_interrupt()  argument
84 	rc = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx);  in xenvif_handle_tx_interrupt()
86 		napi_schedule(&queue->napi);  in xenvif_handle_tx_interrupt()
92 	struct xenvif_queue *queue = dev_id;  in xenvif_tx_interrupt()  local
95 	old = atomic_fetch_or(NETBK_TX_EOI, &queue->eoi_pending);  in xenvif_tx_interrupt()
98 	if (!xenvif_handle_tx_interrupt(queue)) {  in xenvif_tx_interrupt()
99 		atomic_andnot(NETBK_TX_EOI, &queue->eoi_pending);  in xenvif_tx_interrupt()
108 	struct xenvif_queue *queue =  in xenvif_poll()  local
116 	if (unlikely(queue->vif->disabled)) {  in xenvif_poll()
121 	work_done = xenvif_tx_action(queue, budget);  in xenvif_poll()
125 		/* If the queue is rate-limited, it shall be  in xenvif_poll()
128 		if (likely(!queue->rate_limited))  in xenvif_poll()
129 			xenvif_napi_schedule_or_enable_events(queue);  in xenvif_poll()
135 static bool xenvif_handle_rx_interrupt(struct xenvif_queue *queue)  in xenvif_handle_rx_interrupt()  argument
139 	rc = xenvif_have_rx_work(queue, false);  in xenvif_handle_rx_interrupt()
141 		xenvif_kick_thread(queue);  in xenvif_handle_rx_interrupt()
147 	struct xenvif_queue *queue = dev_id;  in xenvif_rx_interrupt()  local
150 	old = atomic_fetch_or(NETBK_RX_EOI, &queue->eoi_pending);  in xenvif_rx_interrupt()
153 	if (!xenvif_handle_rx_interrupt(queue)) {  in xenvif_rx_interrupt()
154 		atomic_andnot(NETBK_RX_EOI, &queue->eoi_pending);  in xenvif_rx_interrupt()
163 	struct xenvif_queue *queue = dev_id;  in xenvif_interrupt()  local
166 	old = atomic_fetch_or(NETBK_COMMON_EOI, &queue->eoi_pending);  in xenvif_interrupt()
170 	if ((!xenvif_handle_tx_interrupt(queue) |  in xenvif_interrupt()
171 	     !xenvif_handle_rx_interrupt(queue))) {  in xenvif_interrupt()
172 		atomic_andnot(NETBK_COMMON_EOI, &queue->eoi_pending);  in xenvif_interrupt()
179 int xenvif_queue_stopped(struct xenvif_queue *queue)  in xenvif_queue_stopped()  argument
181 	struct net_device *dev = queue->vif->dev;  in xenvif_queue_stopped()
182 	unsigned int id = queue->id;  in xenvif_queue_stopped()
186 void xenvif_wake_queue(struct xenvif_queue *queue)  in xenvif_wake_queue()  argument
188 	struct net_device *dev = queue->vif->dev;  in xenvif_wake_queue()
189 	unsigned int id = queue->id;  in xenvif_wake_queue()
223 	struct xenvif_queue *queue = NULL;  in xenvif_start_xmit()  local
238 	/* Obtain the queue to be used to transmit this packet */  in xenvif_start_xmit()
241 		pr_warn_ratelimited("Invalid queue %hu for packet on interface %s\n",  in xenvif_start_xmit()
245 	queue = &vif->queues[index];  in xenvif_start_xmit()
247 	/* Drop the packet if queue is not ready */  in xenvif_start_xmit()
248 	if (queue->task == NULL ||  in xenvif_start_xmit()
249 	    queue->dealloc_task == NULL ||  in xenvif_start_xmit()
270 	xenvif_rx_queue_tail(queue, skb);  in xenvif_start_xmit()
271 	xenvif_kick_thread(queue);  in xenvif_start_xmit()
284 	struct xenvif_queue *queue = NULL;  in xenvif_get_stats()  local
295 	/* Aggregate tx and rx stats from each queue */  in xenvif_get_stats()
297 		queue = &vif->queues[index];  in xenvif_get_stats()
298 		rx_bytes += queue->stats.rx_bytes;  in xenvif_get_stats()
299 		rx_packets += queue->stats.rx_packets;  in xenvif_get_stats()
300 		tx_bytes += queue->stats.tx_bytes;  in xenvif_get_stats()
301 		tx_packets += queue->stats.tx_packets;  in xenvif_get_stats()
316 	struct xenvif_queue *queue = NULL;  in xenvif_up()  local
321 		queue = &vif->queues[queue_index];  in xenvif_up()
322 		napi_enable(&queue->napi);  in xenvif_up()
323 		enable_irq(queue->tx_irq);  in xenvif_up()
324 		if (queue->tx_irq != queue->rx_irq)  in xenvif_up()
325 			enable_irq(queue->rx_irq);  in xenvif_up()
326 		xenvif_napi_schedule_or_enable_events(queue);  in xenvif_up()
332 	struct xenvif_queue *queue = NULL;  in xenvif_down()  local
337 		queue = &vif->queues[queue_index];  in xenvif_down()
338 		disable_irq(queue->tx_irq);  in xenvif_down()
339 		if (queue->tx_irq != queue->rx_irq)  in xenvif_down()
340 			disable_irq(queue->rx_irq);  in xenvif_down()
341 		napi_disable(&queue->napi);  in xenvif_down()
342 		del_timer_sync(&queue->credit_timeout);  in xenvif_down()
571 int xenvif_init_queue(struct xenvif_queue *queue)  in xenvif_init_queue()  argument
575 	queue->credit_bytes = queue->remaining_credit = ~0UL;  in xenvif_init_queue()
576 	queue->credit_usec  = 0UL;  in xenvif_init_queue()
577 	timer_setup(&queue->credit_timeout, xenvif_tx_credit_callback, 0);  in xenvif_init_queue()
578 	queue->credit_window_start = get_jiffies_64();  in xenvif_init_queue()
580 	queue->rx_queue_max = XENVIF_RX_QUEUE_BYTES;  in xenvif_init_queue()
582 	skb_queue_head_init(&queue->rx_queue);  in xenvif_init_queue()
583 	skb_queue_head_init(&queue->tx_queue);  in xenvif_init_queue()
585 	queue->pending_cons = 0;  in xenvif_init_queue()
586 	queue->pending_prod = MAX_PENDING_REQS;  in xenvif_init_queue()
588 		queue->pending_ring[i] = i;  in xenvif_init_queue()
590 	spin_lock_init(&queue->callback_lock);  in xenvif_init_queue()
591 	spin_lock_init(&queue->response_lock);  in xenvif_init_queue()
598 				 queue->mmap_pages);  in xenvif_init_queue()
600 		netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n");  in xenvif_init_queue()
605 		queue->pending_tx_info[i].callback_struct = (struct ubuf_info)  in xenvif_init_queue()
609 		queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;  in xenvif_init_queue()
682 static void xenvif_disconnect_queue(struct xenvif_queue *queue)  in xenvif_disconnect_queue()  argument
684 	if (queue->task) {  in xenvif_disconnect_queue()
685 		kthread_stop(queue->task);  in xenvif_disconnect_queue()
686 		queue->task = NULL;  in xenvif_disconnect_queue()
689 	if (queue->dealloc_task) {  in xenvif_disconnect_queue()
690 		kthread_stop(queue->dealloc_task);  in xenvif_disconnect_queue()
691 		queue->dealloc_task = NULL;  in xenvif_disconnect_queue()
694 	if (queue->napi.poll) {  in xenvif_disconnect_queue()
695 		netif_napi_del(&queue->napi);  in xenvif_disconnect_queue()
696 		queue->napi.poll = NULL;  in xenvif_disconnect_queue()
699 	if (queue->tx_irq) {  in xenvif_disconnect_queue()
700 		unbind_from_irqhandler(queue->tx_irq, queue);  in xenvif_disconnect_queue()
701 		if (queue->tx_irq == queue->rx_irq)  in xenvif_disconnect_queue()
702 			queue->rx_irq = 0;  in xenvif_disconnect_queue()
703 		queue->tx_irq = 0;  in xenvif_disconnect_queue()
706 	if (queue->rx_irq) {  in xenvif_disconnect_queue()
707 		unbind_from_irqhandler(queue->rx_irq, queue);  in xenvif_disconnect_queue()
708 		queue->rx_irq = 0;  in xenvif_disconnect_queue()
711 	xenvif_unmap_frontend_data_rings(queue);  in xenvif_disconnect_queue()
714 int xenvif_connect_data(struct xenvif_queue *queue,  in xenvif_connect_data()  argument
723 	BUG_ON(queue->tx_irq);  in xenvif_connect_data()
724 	BUG_ON(queue->task);  in xenvif_connect_data()
725 	BUG_ON(queue->dealloc_task);  in xenvif_connect_data()
727 	err = xenvif_map_frontend_data_rings(queue, tx_ring_ref,  in xenvif_connect_data()
732 	init_waitqueue_head(&queue->wq);  in xenvif_connect_data()
733 	init_waitqueue_head(&queue->dealloc_wq);  in xenvif_connect_data()
734 	atomic_set(&queue->inflight_packets, 0);  in xenvif_connect_data()
736 	netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll,  in xenvif_connect_data()
739 	queue->stalled = true;  in xenvif_connect_data()
741 	task = kthread_run(xenvif_kthread_guest_rx, queue,  in xenvif_connect_data()
742 			   "%s-guest-rx", queue->name);  in xenvif_connect_data()
745 	queue->task = task;  in xenvif_connect_data()
747 	task = kthread_run(xenvif_dealloc_kthread, queue,  in xenvif_connect_data()
748 			   "%s-dealloc", queue->name);  in xenvif_connect_data()
751 	queue->dealloc_task = task;  in xenvif_connect_data()
756 			queue->vif->domid, tx_evtchn, xenvif_interrupt, 0,  in xenvif_connect_data()
757 			queue->name, queue);  in xenvif_connect_data()
760 		queue->tx_irq = queue->rx_irq = err;  in xenvif_connect_data()
761 		disable_irq(queue->tx_irq);  in xenvif_connect_data()
764 		snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),  in xenvif_connect_data()
765 			 "%s-tx", queue->name);  in xenvif_connect_data()
767 			queue->vif->domid, tx_evtchn, xenvif_tx_interrupt, 0,  in xenvif_connect_data()
768 			queue->tx_irq_name, queue);  in xenvif_connect_data()
771 		queue->tx_irq = err;  in xenvif_connect_data()
772 		disable_irq(queue->tx_irq);  in xenvif_connect_data()
774 		snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),  in xenvif_connect_data()
775 			 "%s-rx", queue->name);  in xenvif_connect_data()
777 			queue->vif->domid, rx_evtchn, xenvif_rx_interrupt, 0,  in xenvif_connect_data()
778 			queue->rx_irq_name, queue);  in xenvif_connect_data()
781 		queue->rx_irq = err;  in xenvif_connect_data()
782 		disable_irq(queue->rx_irq);  in xenvif_connect_data()
788 	pr_warn("Could not allocate kthread for %s\n", queue->name);  in xenvif_connect_data()
791 	xenvif_disconnect_queue(queue);  in xenvif_connect_data()
810 	struct xenvif_queue *queue = NULL;  in xenvif_disconnect_data()  local
817 		queue = &vif->queues[queue_index];  in xenvif_disconnect_data()
819 		xenvif_disconnect_queue(queue);  in xenvif_disconnect_data()
841  * Used for queue teardown from xenvif_free(), and on the
844 void xenvif_deinit_queue(struct xenvif_queue *queue)  in xenvif_deinit_queue()  argument
846 	gnttab_free_pages(MAX_PENDING_REQS, queue->mmap_pages);  in xenvif_deinit_queue()