222 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_txq_notify()  argument
230 	val |= txo->q.id & DB_TXULP_RING_ID_MASK;  in be_txq_notify()
234 	iowrite32(val, adapter->db + txo->db_offset);  in be_txq_notify()
660 	struct be_tx_obj *txo;  in be_get_stats64()  local
680 	for_all_tx_queues(adapter, txo, i) {  in be_get_stats64()
681 		const struct be_tx_stats *tx_stats = tx_stats(txo);  in be_get_stats64()
685 			pkts = tx_stats(txo)->tx_pkts;  in be_get_stats64()
686 			bytes = tx_stats(txo)->tx_bytes;  in be_get_stats64()
746 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)  in be_tx_stats_update()  argument
748 	struct be_tx_stats *stats = tx_stats(txo);  in be_tx_stats_update()
817 static inline bool be_is_txq_full(struct be_tx_obj *txo)  in be_is_txq_full()  argument
819 	return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;  in be_is_txq_full()
822 static inline bool be_can_txq_wake(struct be_tx_obj *txo)  in be_can_txq_wake()  argument
824 	return atomic_read(&txo->q.used) < txo->q.len / 2;  in be_can_txq_wake()
827 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)  in be_is_tx_compl_pending()  argument
829 	return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;  in be_is_tx_compl_pending()
919 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo)  in be_tx_get_wrb_hdr()  argument
921 	u32 head = txo->q.head;  in be_tx_get_wrb_hdr()
923 	queue_head_inc(&txo->q);  in be_tx_get_wrb_hdr()
929 				struct be_tx_obj *txo,  in be_tx_setup_wrb_hdr()  argument
934 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_hdr()
940 	BUG_ON(txo->sent_skb_list[head]);  in be_tx_setup_wrb_hdr()
941 	txo->sent_skb_list[head] = skb;  in be_tx_setup_wrb_hdr()
942 	txo->last_req_hdr = head;  in be_tx_setup_wrb_hdr()
944 	txo->last_req_wrb_cnt = num_frags;  in be_tx_setup_wrb_hdr()
945 	txo->pend_wrb_cnt += num_frags;  in be_tx_setup_wrb_hdr()
949 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,  in be_tx_setup_wrb_frag()  argument
953 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_frag()
965 			    struct be_tx_obj *txo, u32 head, bool map_single,  in be_xmit_restore()  argument
970 	struct be_queue_info *txq = &txo->q;  in be_xmit_restore()
992 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_xmit_enqueue()  argument
1003 	head = be_tx_get_wrb_hdr(txo);  in be_xmit_enqueue()
1012 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
1023 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
1027 	be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);  in be_xmit_enqueue()
1029 	be_tx_stats_update(txo, skb);  in be_xmit_enqueue()
1034 	be_xmit_restore(adapter, txo, head, map_single, copied);  in be_xmit_enqueue()
1225 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)  in be_xmit_flush()  argument
1227 	struct be_queue_info *txq = &txo->q;  in be_xmit_flush()
1228 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);  in be_xmit_flush()
1235 	if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {  in be_xmit_flush()
1239 		txo->pend_wrb_cnt++;  in be_xmit_flush()
1242 		hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<  in be_xmit_flush()
1245 	be_txq_notify(adapter, txo, txo->pend_wrb_cnt);  in be_xmit_flush()
1246 	txo->pend_wrb_cnt = 0;  in be_xmit_flush()
1373 	struct be_tx_obj *txo = &adapter->tx_obj[q_idx];  in be_xmit()  local
1384 	wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1395 		wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1402 	if (be_is_txq_full(txo)) {  in be_xmit()
1404 		tx_stats(txo)->tx_stops++;  in be_xmit()
1408 		be_xmit_flush(adapter, txo);  in be_xmit()
1412 	tx_stats(txo)->tx_drv_drops++;  in be_xmit()
1414 	if (flush && txo->pend_wrb_cnt)  in be_xmit()
1415 		be_xmit_flush(adapter, txo);  in be_xmit()
1424 	struct be_tx_obj *txo;  in be_tx_timeout()  local
1432 	for_all_tx_queues(adapter, txo, i) {  in be_tx_timeout()
1434 			 i, txo->q.head, txo->q.tail,  in be_tx_timeout()
1435 			 atomic_read(&txo->q.used), txo->q.id);  in be_tx_timeout()
1437 		entry = txo->q.dma_mem.va;  in be_tx_timeout()
1447 		entry = txo->cq.dma_mem.va;  in be_tx_timeout()
1449 			 i, txo->cq.head, txo->cq.tail,  in be_tx_timeout()
1450 			 atomic_read(&txo->cq.used));  in be_tx_timeout()
1461 			if (txo->sent_skb_list[j]) {  in be_tx_timeout()
1462 				skb = txo->sent_skb_list[j];  in be_tx_timeout()
2143 	struct be_tx_obj *txo;  in be_get_new_eqd()  local
2164 	for_all_tx_queues_on_eq(adapter, eqo, txo, i) {  in be_get_new_eqd()
2166 			start = u64_stats_fetch_begin_irq(&txo->stats.sync);  in be_get_new_eqd()
2167 			tx_pkts += txo->stats.tx_reqs;  in be_get_new_eqd()
2168 		} while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));  in be_get_new_eqd()
2668 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)  in be_update_tx_err()  argument
2672 		tx_stats(txo)->tx_hdr_parse_err++;  in be_update_tx_err()
2675 		tx_stats(txo)->tx_dma_err++;  in be_update_tx_err()
2678 		tx_stats(txo)->tx_spoof_check_err++;  in be_update_tx_err()
2683 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)  in lancer_update_tx_err()  argument
2687 		tx_stats(txo)->tx_tso_err++;  in lancer_update_tx_err()
2691 		tx_stats(txo)->tx_spoof_check_err++;  in lancer_update_tx_err()
2694 		tx_stats(txo)->tx_qinq_err++;  in lancer_update_tx_err()
2697 		tx_stats(txo)->tx_internal_parity_err++;  in lancer_update_tx_err()
2700 		tx_stats(txo)->tx_dma_err++;  in lancer_update_tx_err()
2703 		tx_stats(txo)->tx_sge_err++;  in lancer_update_tx_err()
2709 						struct be_tx_obj *txo)  in be_tx_compl_get()  argument
2711 	struct be_queue_info *tx_cq = &txo->cq;  in be_tx_compl_get()
2712 	struct be_tx_compl_info *txcp = &txo->txcp;  in be_tx_compl_get()
2727 			lancer_update_tx_err(txo, txcp->status);  in be_tx_compl_get()
2736 			be_update_tx_err(txo, txcp->status);  in be_tx_compl_get()
2749 			       struct be_tx_obj *txo, u16 last_index)  in be_tx_compl_process()  argument
2751 	struct sk_buff **sent_skbs = txo->sent_skb_list;  in be_tx_compl_process()
2752 	struct be_queue_info *txq = &txo->q;  in be_tx_compl_process()
2874 	struct be_tx_obj *txo;  in be_tx_compl_clean()  local
2881 		for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2884 			txq = &txo->q;  in be_tx_compl_clean()
2885 			while ((txcp = be_tx_compl_get(adapter, txo))) {  in be_tx_compl_clean()
2887 					be_tx_compl_process(adapter, txo,  in be_tx_compl_clean()
2892 				be_cq_notify(adapter, txo->cq.id, false, cmpl);  in be_tx_compl_clean()
2896 			if (!be_is_tx_compl_pending(txo))  in be_tx_compl_clean()
2908 	for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2909 		txq = &txo->q;  in be_tx_compl_clean()
2921 			num_wrbs = be_tx_compl_process(adapter, txo, end_idx);  in be_tx_compl_clean()
2924 			txo->pend_wrb_cnt = 0;  in be_tx_compl_clean()
3043 	struct be_tx_obj *txo;  in be_tx_queues_destroy()  local
3046 	for_all_tx_queues(adapter, txo, i) {  in be_tx_queues_destroy()
3047 		q = &txo->q;  in be_tx_queues_destroy()
3052 		q = &txo->cq;  in be_tx_queues_destroy()
3062 	struct be_tx_obj *txo;  in be_tx_qs_create()  local
3068 	for_all_tx_queues(adapter, txo, i) {  in be_tx_qs_create()
3069 		cq = &txo->cq;  in be_tx_qs_create()
3075 		u64_stats_init(&txo->stats.sync);  in be_tx_qs_create()
3076 		u64_stats_init(&txo->stats.sync_compl);  in be_tx_qs_create()
3086 		status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,  in be_tx_qs_create()
3091 		status = be_cmd_txq_create(adapter, txo);  in be_tx_qs_create()
3266 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_process_tx()  argument
3272 	while ((txcp = be_tx_compl_get(adapter, txo))) {  in be_process_tx()
3273 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);  in be_process_tx()
3278 		be_cq_notify(adapter, txo->cq.id, true, work_done);  in be_process_tx()
3279 		atomic_sub(num_wrbs, &txo->q.used);  in be_process_tx()
3284 		    be_can_txq_wake(txo)) {  in be_process_tx()
3288 		u64_stats_update_begin(&tx_stats(txo)->sync_compl);  in be_process_tx()
3289 		tx_stats(txo)->tx_compl += work_done;  in be_process_tx()
3290 		u64_stats_update_end(&tx_stats(txo)->sync_compl);  in be_process_tx()
3300 	struct be_tx_obj *txo;  in be_poll()  local
3305 	for_all_tx_queues_on_eq(adapter, eqo, txo, i)  in be_poll()
3306 		be_process_tx(adapter, txo, i);  in be_poll()
3798 	struct be_tx_obj *txo;  in be_open()  local
3817 	for_all_tx_queues(adapter, txo, i)  in be_open()
3818 		be_cq_notify(adapter, txo->cq.id, true, 0);  in be_open()