299 static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_inc_get()  argument
301 	txq_pcpu->txq_get_index++;  in mvpp2_txq_inc_get()
302 	if (txq_pcpu->txq_get_index == txq_pcpu->size)  in mvpp2_txq_inc_get()
303 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_inc_get()
307 			      struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_inc_put()  argument
313 		txq_pcpu->buffs + txq_pcpu->txq_put_index;  in mvpp2_txq_inc_put()
322 	txq_pcpu->txq_put_index++;  in mvpp2_txq_inc_put()
323 	if (txq_pcpu->txq_put_index == txq_pcpu->size)  in mvpp2_txq_inc_put()
324 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_inc_put()
2504 					    struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_reserved_desc_num_proc()  argument
2510 	if (txq_pcpu->reserved_num >= num)  in mvpp2_txq_reserved_desc_num_proc()
2527 	req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num);  in mvpp2_txq_reserved_desc_num_proc()
2534 	txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req);  in mvpp2_txq_reserved_desc_num_proc()
2537 	if (txq_pcpu->reserved_num < num)  in mvpp2_txq_reserved_desc_num_proc()
2783 				struct mvpp2_txq_pcpu *txq_pcpu, int num)  in mvpp2_txq_bufs_free()  argument
2794 			txq_pcpu->buffs + txq_pcpu->txq_get_index;  in mvpp2_txq_bufs_free()
2796 		if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma) &&  in mvpp2_txq_bufs_free()
2806 		mvpp2_txq_inc_get(txq_pcpu);  in mvpp2_txq_bufs_free()
2831 			   struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_done()  argument
2836 	if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id()))  in mvpp2_txq_done()
2842 	mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done);  in mvpp2_txq_done()
2844 	txq_pcpu->count -= tx_done;  in mvpp2_txq_done()
2847 		if (txq_pcpu->count <= txq_pcpu->wake_threshold)  in mvpp2_txq_done()
2855 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx_done()  local
2863 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_done()
2865 		if (txq_pcpu->count) {  in mvpp2_tx_done()
2866 			mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx_done()
2867 			tx_todo += txq_pcpu->count;  in mvpp2_tx_done()
3068 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_init()  local
3124 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_init()
3125 		txq_pcpu->size = txq->size;  in mvpp2_txq_init()
3126 		txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,  in mvpp2_txq_init()
3127 						sizeof(*txq_pcpu->buffs),  in mvpp2_txq_init()
3129 		if (!txq_pcpu->buffs)  in mvpp2_txq_init()
3132 		txq_pcpu->count = 0;  in mvpp2_txq_init()
3133 		txq_pcpu->reserved_num = 0;  in mvpp2_txq_init()
3134 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_init()
3135 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_init()
3136 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_init()
3138 		txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS;  in mvpp2_txq_init()
3139 		txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2;  in mvpp2_txq_init()
3141 		txq_pcpu->tso_headers =  in mvpp2_txq_init()
3143 					   txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_init()
3144 					   &txq_pcpu->tso_headers_dma,  in mvpp2_txq_init()
3146 		if (!txq_pcpu->tso_headers)  in mvpp2_txq_init()
3157 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_deinit()  local
3161 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_deinit()
3162 		kfree(txq_pcpu->buffs);  in mvpp2_txq_deinit()
3164 		if (txq_pcpu->tso_headers)  in mvpp2_txq_deinit()
3166 					  txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_deinit()
3167 					  txq_pcpu->tso_headers,  in mvpp2_txq_deinit()
3168 					  txq_pcpu->tso_headers_dma);  in mvpp2_txq_deinit()
3170 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_deinit()
3197 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_clean()  local
3231 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_clean()
3234 		mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);  in mvpp2_txq_clean()
3237 		txq_pcpu->count = 0;  in mvpp2_txq_clean()
3238 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_clean()
3239 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_clean()
3614 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_xdp_finish_tx()  local
3619 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_xdp_finish_tx()
3623 	txq_pcpu->reserved_num -= nxmit;  in mvpp2_xdp_finish_tx()
3624 	txq_pcpu->count += nxmit;  in mvpp2_xdp_finish_tx()
3631 	if (txq_pcpu->count >= txq_pcpu->stop_threshold)  in mvpp2_xdp_finish_tx()
3635 	if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)  in mvpp2_xdp_finish_tx()
3636 		mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_xdp_finish_tx()
3647 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_xdp_submit_frame()  local
3655 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_xdp_submit_frame()
3660 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, 1)) {  in mvpp2_xdp_submit_frame()
3697 	mvpp2_txq_inc_put(port, txq_pcpu, xdpf, tx_desc, buf_type);  in mvpp2_xdp_submit_frame()
4053 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in tx_desc_unmap_put()  local
4059 	if (!IS_TSO_HEADER(txq_pcpu, buf_dma_addr))  in tx_desc_unmap_put()
4159 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_frag_process()  local
4187 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx_frag_process()
4191 			mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx_frag_process()
4212 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_hdr()  argument
4223 	addr = txq_pcpu->tso_headers_dma +  in mvpp2_tso_put_hdr()
4224 	       txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tso_put_hdr()
4230 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_hdr()
4237 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_data()  argument
4260 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_data()
4267 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_data()
4274 			struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_tx_tso()  argument
4282 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu,  in mvpp2_tx_tso()
4291 		char *hdr = txq_pcpu->tso_headers +  in mvpp2_tx_tso()
4292 			    txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tx_tso()
4298 		mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);  in mvpp2_tx_tso()
4306 					       txq_pcpu, sz, left, len == 0))  in mvpp2_tx_tso()
4327 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx()  local
4340 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx()
4347 		frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);  in mvpp2_tx()
4354 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) {  in mvpp2_tx()
4383 		mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx()
4388 		mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx()
4402 		txq_pcpu->reserved_num -= frags;  in mvpp2_tx()
4403 		txq_pcpu->count += frags;  in mvpp2_tx()
4410 		if (txq_pcpu->count >= txq_pcpu->stop_threshold)  in mvpp2_tx()
4423 	if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)  in mvpp2_tx()
4424 		mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx()
4427 	if (!port->has_tx_irqs && txq_pcpu->count <= frags &&  in mvpp2_tx()
4428 	    txq_pcpu->count > 0) {  in mvpp2_tx()
5888 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_port_init()  local
5946 			txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_port_init()
5947 			txq_pcpu->thread = thread;  in mvpp2_port_init()