220 static int vector_advancehead(struct vector_queue *qi, int advance)  in vector_advancehead()  argument
224 	qi->head =  in vector_advancehead()
225 		(qi->head + advance)  in vector_advancehead()
226 			% qi->max_depth;  in vector_advancehead()
229 	spin_lock(&qi->tail_lock);  in vector_advancehead()
230 	qi->queue_depth -= advance;  in vector_advancehead()
236 	if (qi->queue_depth == 0) {  in vector_advancehead()
237 		qi->head = 0;  in vector_advancehead()
238 		qi->tail = 0;  in vector_advancehead()
240 	queue_depth = qi->queue_depth;  in vector_advancehead()
241 	spin_unlock(&qi->tail_lock);  in vector_advancehead()
250 static int vector_advancetail(struct vector_queue *qi, int advance)  in vector_advancetail()  argument
254 	qi->tail =  in vector_advancetail()
255 		(qi->tail + advance)  in vector_advancetail()
256 			% qi->max_depth;  in vector_advancetail()
257 	spin_lock(&qi->head_lock);  in vector_advancetail()
258 	qi->queue_depth += advance;  in vector_advancetail()
259 	queue_depth = qi->queue_depth;  in vector_advancetail()
260 	spin_unlock(&qi->head_lock);  in vector_advancetail()
305 static int vector_enqueue(struct vector_queue *qi, struct sk_buff *skb)  in vector_enqueue()  argument
307 	struct vector_private *vp = netdev_priv(qi->dev);  in vector_enqueue()
310 	struct mmsghdr *mmsg_vector = qi->mmsg_vector;  in vector_enqueue()
313 	spin_lock(&qi->tail_lock);  in vector_enqueue()
314 	spin_lock(&qi->head_lock);  in vector_enqueue()
315 	queue_depth = qi->queue_depth;  in vector_enqueue()
316 	spin_unlock(&qi->head_lock);  in vector_enqueue()
321 	if (queue_depth < qi->max_depth) {  in vector_enqueue()
323 		*(qi->skbuff_vector + qi->tail) = skb;  in vector_enqueue()
324 		mmsg_vector += qi->tail;  in vector_enqueue()
335 		queue_depth = vector_advancetail(qi, 1);  in vector_enqueue()
338 	spin_unlock(&qi->tail_lock);  in vector_enqueue()
341 	qi->dev->stats.tx_dropped++;  in vector_enqueue()
345 		netdev_completed_queue(qi->dev, 1, packet_len);  in vector_enqueue()
347 	spin_unlock(&qi->tail_lock);  in vector_enqueue()
351 static int consume_vector_skbs(struct vector_queue *qi, int count)  in consume_vector_skbs()  argument
357 	for (skb_index = qi->head; skb_index < qi->head + count; skb_index++) {  in consume_vector_skbs()
358 		skb = *(qi->skbuff_vector + skb_index);  in consume_vector_skbs()
363 		*(qi->skbuff_vector + skb_index) = NULL;  in consume_vector_skbs()
366 	qi->dev->stats.tx_bytes += bytes_compl;  in consume_vector_skbs()
367 	qi->dev->stats.tx_packets += count;  in consume_vector_skbs()
368 	netdev_completed_queue(qi->dev, count, bytes_compl);  in consume_vector_skbs()
369 	return vector_advancehead(qi, count);  in consume_vector_skbs()
379 static int vector_send(struct vector_queue *qi)  in vector_send()  argument
381 	struct vector_private *vp = netdev_priv(qi->dev);  in vector_send()
383 	int result = 0, send_len, queue_depth = qi->max_depth;  in vector_send()
385 	if (spin_trylock(&qi->head_lock)) {  in vector_send()
386 		if (spin_trylock(&qi->tail_lock)) {  in vector_send()
388 			queue_depth = qi->queue_depth;  in vector_send()
389 			spin_unlock(&qi->tail_lock);  in vector_send()
393 				send_from = qi->mmsg_vector;  in vector_send()
394 				send_from += qi->head;  in vector_send()
396 				if (send_len + qi->head > qi->max_depth)  in vector_send()
397 					send_len = qi->max_depth - qi->head;  in vector_send()
422 						consume_vector_skbs(qi, result);  in vector_send()
432 				netif_trans_update(qi->dev);  in vector_send()
433 				netif_wake_queue(qi->dev);  in vector_send()
443 		spin_unlock(&qi->head_lock);  in vector_send()
454 static void destroy_queue(struct vector_queue *qi)  in destroy_queue()  argument
458 	struct vector_private *vp = netdev_priv(qi->dev);  in destroy_queue()
461 	if (qi == NULL)  in destroy_queue()
466 	if (qi->skbuff_vector != NULL) {  in destroy_queue()
467 		for (i = 0; i < qi->max_depth; i++) {  in destroy_queue()
468 			if (*(qi->skbuff_vector + i) != NULL)  in destroy_queue()
469 				dev_kfree_skb_any(*(qi->skbuff_vector + i));  in destroy_queue()
471 		kfree(qi->skbuff_vector);  in destroy_queue()
474 	if (qi->mmsg_vector != NULL) {  in destroy_queue()
475 		mmsg_vector = qi->mmsg_vector;  in destroy_queue()
476 		for (i = 0; i < qi->max_depth; i++) {  in destroy_queue()
486 		kfree(qi->mmsg_vector);  in destroy_queue()
488 	kfree(qi);  in destroy_queue()
647 static void prep_queue_for_rx(struct vector_queue *qi)  in prep_queue_for_rx()  argument
649 	struct vector_private *vp = netdev_priv(qi->dev);  in prep_queue_for_rx()
650 	struct mmsghdr *mmsg_vector = qi->mmsg_vector;  in prep_queue_for_rx()
651 	void **skbuff_vector = qi->skbuff_vector;  in prep_queue_for_rx()
654 	if (qi->queue_depth == 0)  in prep_queue_for_rx()
656 	for (i = 0; i < qi->queue_depth; i++) {  in prep_queue_for_rx()
666 	qi->queue_depth = 0;  in prep_queue_for_rx()
922 	struct vector_queue *qi = vp->rx_queue;  in vector_mmsg_rx()  local
924 	struct mmsghdr *mmsg_vector = qi->mmsg_vector;  in vector_mmsg_rx()
925 	void **skbuff_vector = qi->skbuff_vector;  in vector_mmsg_rx()
932 	prep_queue_for_rx(qi);  in vector_mmsg_rx()
937 		vp->fds->rx_fd, qi->mmsg_vector, qi->max_depth, 0);  in vector_mmsg_rx()
947 	qi->queue_depth = packet_count;  in vector_mmsg_rx()