58 /* The time that packets can stay on the guest Rx internal queue
106 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,
109 static void make_tx_response(struct xenvif_queue *queue,
113 static void push_tx_responses(struct xenvif_queue *queue);
115 static inline int tx_work_todo(struct xenvif_queue *queue);
117 static inline unsigned long idx_to_pfn(struct xenvif_queue *queue,  in idx_to_pfn()  argument
120 	return page_to_pfn(queue->mmap_pages[idx]);  in idx_to_pfn()
123 static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue,  in idx_to_kaddr()  argument
126 	return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx));  in idx_to_kaddr()
159 void xenvif_kick_thread(struct xenvif_queue *queue)  in xenvif_kick_thread()  argument
161 	wake_up(&queue->wq);  in xenvif_kick_thread()
164 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue)  in xenvif_napi_schedule_or_enable_events()  argument
168 	RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do);  in xenvif_napi_schedule_or_enable_events()
171 		napi_schedule(&queue->napi);  in xenvif_napi_schedule_or_enable_events()
173 				     &queue->eoi_pending) &  in xenvif_napi_schedule_or_enable_events()
175 		xen_irq_lateeoi(queue->tx_irq, 0);  in xenvif_napi_schedule_or_enable_events()
178 static void tx_add_credit(struct xenvif_queue *queue)  in tx_add_credit()  argument
186 	max_burst = max(131072UL, queue->credit_bytes);  in tx_add_credit()
189 	max_credit = queue->remaining_credit + queue->credit_bytes;  in tx_add_credit()
190 	if (max_credit < queue->remaining_credit)  in tx_add_credit()
193 	queue->remaining_credit = min(max_credit, max_burst);  in tx_add_credit()
194 	queue->rate_limited = false;  in tx_add_credit()
199 	struct xenvif_queue *queue = from_timer(queue, t, credit_timeout);  in xenvif_tx_credit_callback()  local
200 	tx_add_credit(queue);  in xenvif_tx_credit_callback()
201 	xenvif_napi_schedule_or_enable_events(queue);  in xenvif_tx_credit_callback()
204 static void xenvif_tx_err(struct xenvif_queue *queue,  in xenvif_tx_err()  argument
208 	RING_IDX cons = queue->tx.req_cons;  in xenvif_tx_err()
212 		spin_lock_irqsave(&queue->response_lock, flags);  in xenvif_tx_err()
213 		make_tx_response(queue, txp, extra_count, XEN_NETIF_RSP_ERROR);  in xenvif_tx_err()
214 		push_tx_responses(queue);  in xenvif_tx_err()
215 		spin_unlock_irqrestore(&queue->response_lock, flags);  in xenvif_tx_err()
218 		RING_COPY_REQUEST(&queue->tx, cons++, txp);  in xenvif_tx_err()
221 	queue->tx.req_cons = cons;  in xenvif_tx_err()
228 	/* Disable the vif from queue 0's kthread */  in xenvif_fatal_tx_err()
233 static int xenvif_count_requests(struct xenvif_queue *queue,  in xenvif_count_requests()  argument
239 	RING_IDX cons = queue->tx.req_cons;  in xenvif_count_requests()
251 			netdev_err(queue->vif->dev,  in xenvif_count_requests()
254 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
262 			netdev_err(queue->vif->dev,  in xenvif_count_requests()
265 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
278 				netdev_dbg(queue->vif->dev,  in xenvif_count_requests()
287 		RING_COPY_REQUEST(&queue->tx, cons + slots, txp);  in xenvif_count_requests()
300 				netdev_dbg(queue->vif->dev,  in xenvif_count_requests()
310 			netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %u, size: %u\n",  in xenvif_count_requests()
312 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
324 		xenvif_tx_err(queue, first, extra_count, cons + slots);  in xenvif_count_requests()
338 static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue,  in xenvif_tx_create_map_op()  argument
344 	queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx];  in xenvif_tx_create_map_op()
345 	gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx),  in xenvif_tx_create_map_op()
347 			  txp->gref, queue->vif->domid);  in xenvif_tx_create_map_op()
349 	memcpy(&queue->pending_tx_info[pending_idx].req, txp,  in xenvif_tx_create_map_op()
351 	queue->pending_tx_info[pending_idx].extra_count = extra_count;  in xenvif_tx_create_map_op()
371 static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *queue,  in xenvif_get_requests()  argument
392 		index = pending_index(queue->pending_cons++);  in xenvif_get_requests()
393 		pending_idx = queue->pending_ring[index];  in xenvif_get_requests()
394 		xenvif_tx_create_map_op(queue, pending_idx, txp, 0, gop);  in xenvif_get_requests()
405 			index = pending_index(queue->pending_cons++);  in xenvif_get_requests()
406 			pending_idx = queue->pending_ring[index];  in xenvif_get_requests()
407 			xenvif_tx_create_map_op(queue, pending_idx, txp, 0,  in xenvif_get_requests()
419 static inline void xenvif_grant_handle_set(struct xenvif_queue *queue,  in xenvif_grant_handle_set()  argument
423 	if (unlikely(queue->grant_tx_handle[pending_idx] !=  in xenvif_grant_handle_set()
425 		netdev_err(queue->vif->dev,  in xenvif_grant_handle_set()
430 	queue->grant_tx_handle[pending_idx] = handle;  in xenvif_grant_handle_set()
433 static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue,  in xenvif_grant_handle_reset()  argument
436 	if (unlikely(queue->grant_tx_handle[pending_idx] ==  in xenvif_grant_handle_reset()
438 		netdev_err(queue->vif->dev,  in xenvif_grant_handle_reset()
443 	queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE;  in xenvif_grant_handle_reset()
446 static int xenvif_tx_check_gop(struct xenvif_queue *queue,  in xenvif_tx_check_gop()  argument
470 			netdev_dbg(queue->vif->dev,  in xenvif_tx_check_gop()
477 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
492 			xenvif_grant_handle_set(queue,  in xenvif_tx_check_gop()
497 				xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
503 					xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
506 					xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
514 			netdev_dbg(queue->vif->dev,  in xenvif_tx_check_gop()
521 		xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR);  in xenvif_tx_check_gop()
531 			xenvif_idx_release(queue,  in xenvif_tx_check_gop()
538 			xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
539 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
549 				xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
550 				xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
571 static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb)  in xenvif_fill_frags()  argument
589 				&callback_param(queue, pending_idx);  in xenvif_fill_frags()
591 			callback_param(queue, prev_pending_idx).ctx =  in xenvif_fill_frags()
592 				&callback_param(queue, pending_idx);  in xenvif_fill_frags()
594 		callback_param(queue, pending_idx).ctx = NULL;  in xenvif_fill_frags()
597 		txp = &queue->pending_tx_info[pending_idx].req;  in xenvif_fill_frags()
598 		page = virt_to_page(idx_to_kaddr(queue, pending_idx));  in xenvif_fill_frags()
605 		get_page(queue->mmap_pages[pending_idx]);  in xenvif_fill_frags()
609 static int xenvif_get_extras(struct xenvif_queue *queue,  in xenvif_get_extras()  argument
615 	RING_IDX cons = queue->tx.req_cons;  in xenvif_get_extras()
619 			netdev_err(queue->vif->dev, "Missing extra info\n");  in xenvif_get_extras()
620 			xenvif_fatal_tx_err(queue->vif);  in xenvif_get_extras()
624 		RING_COPY_REQUEST(&queue->tx, cons, &extra);  in xenvif_get_extras()
626 		queue->tx.req_cons = ++cons;  in xenvif_get_extras()
631 			netdev_err(queue->vif->dev,  in xenvif_get_extras()
633 			xenvif_fatal_tx_err(queue->vif);  in xenvif_get_extras()
672 static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb)  in checksum_setup()  argument
682 		queue->stats.rx_gso_checksum_fixup++;  in checksum_setup()
694 static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size)  in tx_credit_exceeded()  argument
697 	u64 next_credit = queue->credit_window_start +  in tx_credit_exceeded()
698 		msecs_to_jiffies(queue->credit_usec / 1000);  in tx_credit_exceeded()
701 	if (timer_pending(&queue->credit_timeout)) {  in tx_credit_exceeded()
702 		queue->rate_limited = true;  in tx_credit_exceeded()
708 		queue->credit_window_start = now;  in tx_credit_exceeded()
709 		tx_add_credit(queue);  in tx_credit_exceeded()
713 	if (size > queue->remaining_credit) {  in tx_credit_exceeded()
714 		mod_timer(&queue->credit_timeout,  in tx_credit_exceeded()
716 		queue->credit_window_start = next_credit;  in tx_credit_exceeded()
717 		queue->rate_limited = true;  in tx_credit_exceeded()
784 	/* No need for locking or RCU here. NAPI poll and TX queue  in xenvif_mcast_addr_list_free()
799 static void xenvif_tx_build_gops(struct xenvif_queue *queue,  in xenvif_tx_build_gops()  argument
804 	struct gnttab_map_grant_ref *gop = queue->tx_map_ops;  in xenvif_tx_build_gops()
809 	while (skb_queue_len(&queue->tx_queue) < budget) {  in xenvif_tx_build_gops()
820 		if (queue->tx.sring->req_prod - queue->tx.req_cons >  in xenvif_tx_build_gops()
822 			netdev_err(queue->vif->dev,  in xenvif_tx_build_gops()
825 				   queue->tx.sring->req_prod, queue->tx.req_cons,  in xenvif_tx_build_gops()
827 			xenvif_fatal_tx_err(queue->vif);  in xenvif_tx_build_gops()
831 		work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx);  in xenvif_tx_build_gops()
835 		idx = queue->tx.req_cons;  in xenvif_tx_build_gops()
837 		RING_COPY_REQUEST(&queue->tx, idx, &txreq);  in xenvif_tx_build_gops()
840 		if (txreq.size > queue->remaining_credit &&  in xenvif_tx_build_gops()
841 		    tx_credit_exceeded(queue, txreq.size))  in xenvif_tx_build_gops()
844 		queue->remaining_credit -= txreq.size;  in xenvif_tx_build_gops()
847 		queue->tx.req_cons = ++idx;  in xenvif_tx_build_gops()
852 			work_to_do = xenvif_get_extras(queue, extras,  in xenvif_tx_build_gops()
855 			idx = queue->tx.req_cons;  in xenvif_tx_build_gops()
864 			ret = xenvif_mcast_add(queue->vif, extra->u.mcast.addr);  in xenvif_tx_build_gops()
866 			make_tx_response(queue, &txreq, extra_count,  in xenvif_tx_build_gops()
870 			push_tx_responses(queue);  in xenvif_tx_build_gops()
878 			xenvif_mcast_del(queue->vif, extra->u.mcast.addr);  in xenvif_tx_build_gops()
880 			make_tx_response(queue, &txreq, extra_count,  in xenvif_tx_build_gops()
882 			push_tx_responses(queue);  in xenvif_tx_build_gops()
886 		ret = xenvif_count_requests(queue, &txreq, extra_count,  in xenvif_tx_build_gops()
894 			netdev_dbg(queue->vif->dev,  in xenvif_tx_build_gops()
896 			xenvif_tx_err(queue, &txreq, extra_count, idx);  in xenvif_tx_build_gops()
902 			netdev_err(queue->vif->dev,  in xenvif_tx_build_gops()
906 			xenvif_fatal_tx_err(queue->vif);  in xenvif_tx_build_gops()
910 		index = pending_index(queue->pending_cons);  in xenvif_tx_build_gops()
911 		pending_idx = queue->pending_ring[index];  in xenvif_tx_build_gops()
919 			netdev_dbg(queue->vif->dev,  in xenvif_tx_build_gops()
921 			xenvif_tx_err(queue, &txreq, extra_count, idx);  in xenvif_tx_build_gops()
941 				xenvif_tx_err(queue, &txreq, extra_count, idx);  in xenvif_tx_build_gops()
943 					netdev_err(queue->vif->dev,  in xenvif_tx_build_gops()
953 			if (xenvif_set_skb_gso(queue->vif, skb, gso)) {  in xenvif_tx_build_gops()
992 		queue->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref;  in xenvif_tx_build_gops()
993 		queue->tx_copy_ops[*copy_ops].source.domid = queue->vif->domid;  in xenvif_tx_build_gops()
994 		queue->tx_copy_ops[*copy_ops].source.offset = txreq.offset;  in xenvif_tx_build_gops()
996 		queue->tx_copy_ops[*copy_ops].dest.u.gmfn =  in xenvif_tx_build_gops()
998 		queue->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF;  in xenvif_tx_build_gops()
999 		queue->tx_copy_ops[*copy_ops].dest.offset =  in xenvif_tx_build_gops()
1002 		queue->tx_copy_ops[*copy_ops].len = data_len;  in xenvif_tx_build_gops()
1003 		queue->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref;  in xenvif_tx_build_gops()
1010 			xenvif_tx_create_map_op(queue, pending_idx, &txreq,  in xenvif_tx_build_gops()
1016 			memcpy(&queue->pending_tx_info[pending_idx].req,  in xenvif_tx_build_gops()
1018 			queue->pending_tx_info[pending_idx].extra_count =  in xenvif_tx_build_gops()
1022 		queue->pending_cons++;  in xenvif_tx_build_gops()
1024 		gop = xenvif_get_requests(queue, skb, txfrags, gop,  in xenvif_tx_build_gops()
1027 		__skb_queue_tail(&queue->tx_queue, skb);  in xenvif_tx_build_gops()
1029 		queue->tx.req_cons = idx;  in xenvif_tx_build_gops()
1031 		if (((gop-queue->tx_map_ops) >= ARRAY_SIZE(queue->tx_map_ops)) ||  in xenvif_tx_build_gops()
1032 		    (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops)))  in xenvif_tx_build_gops()
1036 	(*map_ops) = gop - queue->tx_map_ops;  in xenvif_tx_build_gops()
1043 static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb)  in xenvif_handle_frag_list()  argument
1051 	queue->stats.tx_zerocopy_sent += 2;  in xenvif_handle_frag_list()
1052 	queue->stats.tx_frag_overflow++;  in xenvif_handle_frag_list()
1054 	xenvif_fill_frags(queue, nskb);  in xenvif_handle_frag_list()
1093 	atomic_inc(&queue->inflight_packets);  in xenvif_handle_frag_list()
1105 static int xenvif_tx_submit(struct xenvif_queue *queue)  in xenvif_tx_submit()  argument
1107 	struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops;  in xenvif_tx_submit()
1108 	struct gnttab_copy *gop_copy = queue->tx_copy_ops;  in xenvif_tx_submit()
1112 	while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) {  in xenvif_tx_submit()
1118 		txp = &queue->pending_tx_info[pending_idx].req;  in xenvif_tx_submit()
1121 		if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) {  in xenvif_tx_submit()
1137 		callback_param(queue, pending_idx).ctx = NULL;  in xenvif_tx_submit()
1144 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_submit()
1153 		xenvif_fill_frags(queue, skb);  in xenvif_tx_submit()
1157 			xenvif_skb_zerocopy_prepare(queue, nskb);  in xenvif_tx_submit()
1158 			if (xenvif_handle_frag_list(queue, skb)) {  in xenvif_tx_submit()
1160 					netdev_err(queue->vif->dev,  in xenvif_tx_submit()
1162 				xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1171 		skb->dev      = queue->vif->dev;  in xenvif_tx_submit()
1175 		if (checksum_setup(queue, skb)) {  in xenvif_tx_submit()
1176 			netdev_dbg(queue->vif->dev,  in xenvif_tx_submit()
1180 				xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1210 		queue->stats.rx_bytes += skb->len;  in xenvif_tx_submit()
1211 		queue->stats.rx_packets++;  in xenvif_tx_submit()
1221 			xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1222 			queue->stats.tx_zerocopy_sent++;  in xenvif_tx_submit()
1235 	struct xenvif_queue *queue = ubuf_to_queue(ubuf);  in xenvif_zerocopy_callback()  local
1240 	spin_lock_irqsave(&queue->callback_lock, flags);  in xenvif_zerocopy_callback()
1244 		BUG_ON(queue->dealloc_prod - queue->dealloc_cons >=  in xenvif_zerocopy_callback()
1246 		index = pending_index(queue->dealloc_prod);  in xenvif_zerocopy_callback()
1247 		queue->dealloc_ring[index] = pending_idx;  in xenvif_zerocopy_callback()
1252 		queue->dealloc_prod++;  in xenvif_zerocopy_callback()
1254 	spin_unlock_irqrestore(&queue->callback_lock, flags);  in xenvif_zerocopy_callback()
1257 		queue->stats.tx_zerocopy_success++;  in xenvif_zerocopy_callback()
1259 		queue->stats.tx_zerocopy_fail++;  in xenvif_zerocopy_callback()
1260 	xenvif_skb_zerocopy_complete(queue);  in xenvif_zerocopy_callback()
1263 static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)  in xenvif_tx_dealloc_action()  argument
1270 	dc = queue->dealloc_cons;  in xenvif_tx_dealloc_action()
1271 	gop = queue->tx_unmap_ops;  in xenvif_tx_dealloc_action()
1275 		dp = queue->dealloc_prod;  in xenvif_tx_dealloc_action()
1283 			BUG_ON(gop - queue->tx_unmap_ops >= MAX_PENDING_REQS);  in xenvif_tx_dealloc_action()
1285 				queue->dealloc_ring[pending_index(dc++)];  in xenvif_tx_dealloc_action()
1287 			pending_idx_release[gop - queue->tx_unmap_ops] =  in xenvif_tx_dealloc_action()
1289 			queue->pages_to_unmap[gop - queue->tx_unmap_ops] =  in xenvif_tx_dealloc_action()
1290 				queue->mmap_pages[pending_idx];  in xenvif_tx_dealloc_action()
1292 					    idx_to_kaddr(queue, pending_idx),  in xenvif_tx_dealloc_action()
1294 					    queue->grant_tx_handle[pending_idx]);  in xenvif_tx_dealloc_action()
1295 			xenvif_grant_handle_reset(queue, pending_idx);  in xenvif_tx_dealloc_action()
1299 	} while (dp != queue->dealloc_prod);  in xenvif_tx_dealloc_action()
1301 	queue->dealloc_cons = dc;  in xenvif_tx_dealloc_action()
1303 	if (gop - queue->tx_unmap_ops > 0) {  in xenvif_tx_dealloc_action()
1305 		ret = gnttab_unmap_refs(queue->tx_unmap_ops,  in xenvif_tx_dealloc_action()
1307 					queue->pages_to_unmap,  in xenvif_tx_dealloc_action()
1308 					gop - queue->tx_unmap_ops);  in xenvif_tx_dealloc_action()
1310 			netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tu ret %d\n",  in xenvif_tx_dealloc_action()
1311 				   gop - queue->tx_unmap_ops, ret);  in xenvif_tx_dealloc_action()
1312 			for (i = 0; i < gop - queue->tx_unmap_ops; ++i) {  in xenvif_tx_dealloc_action()
1314 					netdev_err(queue->vif->dev,  in xenvif_tx_dealloc_action()
1324 	for (i = 0; i < gop - queue->tx_unmap_ops; ++i)  in xenvif_tx_dealloc_action()
1325 		xenvif_idx_release(queue, pending_idx_release[i],  in xenvif_tx_dealloc_action()
1331 int xenvif_tx_action(struct xenvif_queue *queue, int budget)  in xenvif_tx_action()  argument
1336 	if (unlikely(!tx_work_todo(queue)))  in xenvif_tx_action()
1339 	xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops);  in xenvif_tx_action()
1344 	gnttab_batch_copy(queue->tx_copy_ops, nr_cops);  in xenvif_tx_action()
1346 		ret = gnttab_map_refs(queue->tx_map_ops,  in xenvif_tx_action()
1348 				      queue->pages_to_map,  in xenvif_tx_action()
1353 	work_done = xenvif_tx_submit(queue);  in xenvif_tx_action()
1358 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,  in xenvif_idx_release()  argument
1365 	pending_tx_info = &queue->pending_tx_info[pending_idx];  in xenvif_idx_release()
1367 	spin_lock_irqsave(&queue->response_lock, flags);  in xenvif_idx_release()
1369 	make_tx_response(queue, &pending_tx_info->req,  in xenvif_idx_release()
1376 	index = pending_index(queue->pending_prod++);  in xenvif_idx_release()
1377 	queue->pending_ring[index] = pending_idx;  in xenvif_idx_release()
1379 	push_tx_responses(queue);  in xenvif_idx_release()
1381 	spin_unlock_irqrestore(&queue->response_lock, flags);  in xenvif_idx_release()
1385 static void make_tx_response(struct xenvif_queue *queue,  in make_tx_response()  argument
1390 	RING_IDX i = queue->tx.rsp_prod_pvt;  in make_tx_response()
1393 	resp = RING_GET_RESPONSE(&queue->tx, i);  in make_tx_response()
1398 		RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL;  in make_tx_response()
1400 	queue->tx.rsp_prod_pvt = ++i;  in make_tx_response()
1403 static void push_tx_responses(struct xenvif_queue *queue)  in push_tx_responses()  argument
1407 	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify);  in push_tx_responses()
1409 		notify_remote_via_irq(queue->tx_irq);  in push_tx_responses()
1412 void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx)  in xenvif_idx_unmap()  argument
1418 			    idx_to_kaddr(queue, pending_idx),  in xenvif_idx_unmap()
1420 			    queue->grant_tx_handle[pending_idx]);  in xenvif_idx_unmap()
1421 	xenvif_grant_handle_reset(queue, pending_idx);  in xenvif_idx_unmap()
1424 				&queue->mmap_pages[pending_idx], 1);  in xenvif_idx_unmap()
1426 		netdev_err(queue->vif->dev,  in xenvif_idx_unmap()
1437 static inline int tx_work_todo(struct xenvif_queue *queue)  in tx_work_todo()  argument
1439 	if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx)))  in tx_work_todo()
1445 static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue)  in tx_dealloc_work_todo()  argument
1447 	return queue->dealloc_cons != queue->dealloc_prod;  in tx_dealloc_work_todo()
1450 void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue)  in xenvif_unmap_frontend_data_rings()  argument
1452 	if (queue->tx.sring)  in xenvif_unmap_frontend_data_rings()
1453 		xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),  in xenvif_unmap_frontend_data_rings()
1454 					queue->tx.sring);  in xenvif_unmap_frontend_data_rings()
1455 	if (queue->rx.sring)  in xenvif_unmap_frontend_data_rings()
1456 		xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),  in xenvif_unmap_frontend_data_rings()
1457 					queue->rx.sring);  in xenvif_unmap_frontend_data_rings()
1460 int xenvif_map_frontend_data_rings(struct xenvif_queue *queue,  in xenvif_map_frontend_data_rings()  argument
1470 	err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),  in xenvif_map_frontend_data_rings()
1479 	BACK_RING_ATTACH(&queue->tx, txs, rsp_prod, XEN_PAGE_SIZE);  in xenvif_map_frontend_data_rings()
1482 	if (req_prod - rsp_prod > RING_SIZE(&queue->tx))  in xenvif_map_frontend_data_rings()
1485 	err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),  in xenvif_map_frontend_data_rings()
1494 	BACK_RING_ATTACH(&queue->rx, rxs, rsp_prod, XEN_PAGE_SIZE);  in xenvif_map_frontend_data_rings()
1497 	if (req_prod - rsp_prod > RING_SIZE(&queue->rx))  in xenvif_map_frontend_data_rings()
1503 	xenvif_unmap_frontend_data_rings(queue);  in xenvif_map_frontend_data_rings()
1507 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue)  in xenvif_dealloc_kthread_should_stop()  argument
1513 		!atomic_read(&queue->inflight_packets);  in xenvif_dealloc_kthread_should_stop()
1518 	struct xenvif_queue *queue = data;  in xenvif_dealloc_kthread()  local
1521 		wait_event_interruptible(queue->dealloc_wq,  in xenvif_dealloc_kthread()
1522 					 tx_dealloc_work_todo(queue) ||  in xenvif_dealloc_kthread()
1523 					 xenvif_dealloc_kthread_should_stop(queue));  in xenvif_dealloc_kthread()
1524 		if (xenvif_dealloc_kthread_should_stop(queue))  in xenvif_dealloc_kthread()
1527 		xenvif_tx_dealloc_action(queue);  in xenvif_dealloc_kthread()
1532 	if (tx_dealloc_work_todo(queue))  in xenvif_dealloc_kthread()
1533 		xenvif_tx_dealloc_action(queue);  in xenvif_dealloc_kthread()