272 static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_inc_get()  argument
274 	txq_pcpu->txq_get_index++;  in mvpp2_txq_inc_get()
275 	if (txq_pcpu->txq_get_index == txq_pcpu->size)  in mvpp2_txq_inc_get()
276 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_inc_get()
280 			      struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_inc_put()  argument
285 		txq_pcpu->buffs + txq_pcpu->txq_put_index;  in mvpp2_txq_inc_put()
290 	txq_pcpu->txq_put_index++;  in mvpp2_txq_inc_put()
291 	if (txq_pcpu->txq_put_index == txq_pcpu->size)  in mvpp2_txq_inc_put()
292 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_inc_put()
1968 					    struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_reserved_desc_num_proc()  argument
1974 	if (txq_pcpu->reserved_num >= num)  in mvpp2_txq_reserved_desc_num_proc()
1991 	req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num);  in mvpp2_txq_reserved_desc_num_proc()
1998 	txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req);  in mvpp2_txq_reserved_desc_num_proc()
2001 	if (txq_pcpu->reserved_num < num)  in mvpp2_txq_reserved_desc_num_proc()
2232 				struct mvpp2_txq_pcpu *txq_pcpu, int num)  in mvpp2_txq_bufs_free()  argument
2238 			txq_pcpu->buffs + txq_pcpu->txq_get_index;  in mvpp2_txq_bufs_free()
2240 		if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma))  in mvpp2_txq_bufs_free()
2246 		mvpp2_txq_inc_get(txq_pcpu);  in mvpp2_txq_bufs_free()
2268 			   struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_done()  argument
2273 	if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id()))  in mvpp2_txq_done()
2279 	mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done);  in mvpp2_txq_done()
2281 	txq_pcpu->count -= tx_done;  in mvpp2_txq_done()
2284 		if (txq_pcpu->count <= txq_pcpu->wake_threshold)  in mvpp2_txq_done()
2292 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx_done()  local
2300 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_done()
2302 		if (txq_pcpu->count) {  in mvpp2_tx_done()
2303 			mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx_done()
2304 			tx_todo += txq_pcpu->count;  in mvpp2_tx_done()
2459 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_init()  local
2515 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_init()
2516 		txq_pcpu->size = txq->size;  in mvpp2_txq_init()
2517 		txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,  in mvpp2_txq_init()
2518 						sizeof(*txq_pcpu->buffs),  in mvpp2_txq_init()
2520 		if (!txq_pcpu->buffs)  in mvpp2_txq_init()
2523 		txq_pcpu->count = 0;  in mvpp2_txq_init()
2524 		txq_pcpu->reserved_num = 0;  in mvpp2_txq_init()
2525 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_init()
2526 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_init()
2527 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_init()
2529 		txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS;  in mvpp2_txq_init()
2530 		txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2;  in mvpp2_txq_init()
2532 		txq_pcpu->tso_headers =  in mvpp2_txq_init()
2534 					   txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_init()
2535 					   &txq_pcpu->tso_headers_dma,  in mvpp2_txq_init()
2537 		if (!txq_pcpu->tso_headers)  in mvpp2_txq_init()
2548 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_deinit()  local
2552 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_deinit()
2553 		kfree(txq_pcpu->buffs);  in mvpp2_txq_deinit()
2555 		if (txq_pcpu->tso_headers)  in mvpp2_txq_deinit()
2557 					  txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_deinit()
2558 					  txq_pcpu->tso_headers,  in mvpp2_txq_deinit()
2559 					  txq_pcpu->tso_headers_dma);  in mvpp2_txq_deinit()
2561 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_deinit()
2588 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_clean()  local
2622 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_clean()
2625 		mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);  in mvpp2_txq_clean()
2628 		txq_pcpu->count = 0;  in mvpp2_txq_clean()
2629 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_clean()
2630 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_clean()
3021 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in tx_desc_unmap_put()  local
3027 	if (!IS_TSO_HEADER(txq_pcpu, buf_dma_addr))  in tx_desc_unmap_put()
3039 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_frag_process()  local
3066 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);  in mvpp2_tx_frag_process()
3070 			mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);  in mvpp2_tx_frag_process()
3091 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_hdr()  argument
3101 	addr = txq_pcpu->tso_headers_dma +  in mvpp2_tso_put_hdr()
3102 	       txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tso_put_hdr()
3108 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);  in mvpp2_tso_put_hdr()
3115 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_data()  argument
3137 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);  in mvpp2_tso_put_data()
3144 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);  in mvpp2_tso_put_data()
3151 			struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_tx_tso()  argument
3160 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu,  in mvpp2_tx_tso()
3168 		char *hdr = txq_pcpu->tso_headers +  in mvpp2_tx_tso()
3169 			    txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tx_tso()
3175 		mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);  in mvpp2_tx_tso()
3183 					       txq_pcpu, sz, left, len == 0))  in mvpp2_tx_tso()
3204 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx()  local
3217 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx()
3224 		frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);  in mvpp2_tx()
3231 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) {  in mvpp2_tx()
3257 		mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);  in mvpp2_tx()
3262 		mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);  in mvpp2_tx()
3276 		txq_pcpu->reserved_num -= frags;  in mvpp2_tx()
3277 		txq_pcpu->count += frags;  in mvpp2_tx()
3284 		if (txq_pcpu->count >= txq_pcpu->stop_threshold)  in mvpp2_tx()
3297 	if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)  in mvpp2_tx()
3298 		mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx()
3301 	if (!port->has_tx_irqs && txq_pcpu->count <= frags &&  in mvpp2_tx()
3302 	    txq_pcpu->count > 0) {  in mvpp2_tx()
4542 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_port_init()  local
4588 			txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_port_init()
4589 			txq_pcpu->thread = thread;  in mvpp2_port_init()