221 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_txq_notify()  argument
229 	val |= txo->q.id & DB_TXULP_RING_ID_MASK;  in be_txq_notify()
233 	iowrite32(val, adapter->db + txo->db_offset);  in be_txq_notify()
659 	struct be_tx_obj *txo;  in be_get_stats64()  local
679 	for_all_tx_queues(adapter, txo, i) {  in be_get_stats64()
680 		const struct be_tx_stats *tx_stats = tx_stats(txo);  in be_get_stats64()
684 			pkts = tx_stats(txo)->tx_pkts;  in be_get_stats64()
685 			bytes = tx_stats(txo)->tx_bytes;  in be_get_stats64()
745 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)  in be_tx_stats_update()  argument
747 	struct be_tx_stats *stats = tx_stats(txo);  in be_tx_stats_update()
816 static inline bool be_is_txq_full(struct be_tx_obj *txo)  in be_is_txq_full()  argument
818 	return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;  in be_is_txq_full()
821 static inline bool be_can_txq_wake(struct be_tx_obj *txo)  in be_can_txq_wake()  argument
823 	return atomic_read(&txo->q.used) < txo->q.len / 2;  in be_can_txq_wake()
826 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)  in be_is_tx_compl_pending()  argument
828 	return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;  in be_is_tx_compl_pending()
918 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo)  in be_tx_get_wrb_hdr()  argument
920 	u32 head = txo->q.head;  in be_tx_get_wrb_hdr()
922 	queue_head_inc(&txo->q);  in be_tx_get_wrb_hdr()
928 				struct be_tx_obj *txo,  in be_tx_setup_wrb_hdr()  argument
933 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_hdr()
939 	BUG_ON(txo->sent_skb_list[head]);  in be_tx_setup_wrb_hdr()
940 	txo->sent_skb_list[head] = skb;  in be_tx_setup_wrb_hdr()
941 	txo->last_req_hdr = head;  in be_tx_setup_wrb_hdr()
943 	txo->last_req_wrb_cnt = num_frags;  in be_tx_setup_wrb_hdr()
944 	txo->pend_wrb_cnt += num_frags;  in be_tx_setup_wrb_hdr()
948 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,  in be_tx_setup_wrb_frag()  argument
952 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_frag()
964 			    struct be_tx_obj *txo, u32 head, bool map_single,  in be_xmit_restore()  argument
969 	struct be_queue_info *txq = &txo->q;  in be_xmit_restore()
991 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_xmit_enqueue()  argument
1002 	head = be_tx_get_wrb_hdr(txo);  in be_xmit_enqueue()
1011 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
1022 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
1026 	be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);  in be_xmit_enqueue()
1028 	be_tx_stats_update(txo, skb);  in be_xmit_enqueue()
1033 	be_xmit_restore(adapter, txo, head, map_single, copied);  in be_xmit_enqueue()
1224 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)  in be_xmit_flush()  argument
1226 	struct be_queue_info *txq = &txo->q;  in be_xmit_flush()
1227 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);  in be_xmit_flush()
1234 	if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {  in be_xmit_flush()
1238 		txo->pend_wrb_cnt++;  in be_xmit_flush()
1241 		hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<  in be_xmit_flush()
1244 	be_txq_notify(adapter, txo, txo->pend_wrb_cnt);  in be_xmit_flush()
1245 	txo->pend_wrb_cnt = 0;  in be_xmit_flush()
1372 	struct be_tx_obj *txo = &adapter->tx_obj[q_idx];  in be_xmit()  local
1383 	wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1394 		wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1401 	if (be_is_txq_full(txo)) {  in be_xmit()
1403 		tx_stats(txo)->tx_stops++;  in be_xmit()
1407 		be_xmit_flush(adapter, txo);  in be_xmit()
1411 	tx_stats(txo)->tx_drv_drops++;  in be_xmit()
1413 	if (flush && txo->pend_wrb_cnt)  in be_xmit()
1414 		be_xmit_flush(adapter, txo);  in be_xmit()
1423 	struct be_tx_obj *txo;  in be_tx_timeout()  local
1431 	for_all_tx_queues(adapter, txo, i) {  in be_tx_timeout()
1433 			 i, txo->q.head, txo->q.tail,  in be_tx_timeout()
1434 			 atomic_read(&txo->q.used), txo->q.id);  in be_tx_timeout()
1436 		entry = txo->q.dma_mem.va;  in be_tx_timeout()
1446 		entry = txo->cq.dma_mem.va;  in be_tx_timeout()
1448 			 i, txo->cq.head, txo->cq.tail,  in be_tx_timeout()
1449 			 atomic_read(&txo->cq.used));  in be_tx_timeout()
1460 			if (txo->sent_skb_list[j]) {  in be_tx_timeout()
1461 				skb = txo->sent_skb_list[j];  in be_tx_timeout()
2142 	struct be_tx_obj *txo;  in be_get_new_eqd()  local
2163 	for_all_tx_queues_on_eq(adapter, eqo, txo, i) {  in be_get_new_eqd()
2165 			start = u64_stats_fetch_begin_irq(&txo->stats.sync);  in be_get_new_eqd()
2166 			tx_pkts += txo->stats.tx_reqs;  in be_get_new_eqd()
2167 		} while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));  in be_get_new_eqd()
2667 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)  in be_update_tx_err()  argument
2671 		tx_stats(txo)->tx_hdr_parse_err++;  in be_update_tx_err()
2674 		tx_stats(txo)->tx_dma_err++;  in be_update_tx_err()
2677 		tx_stats(txo)->tx_spoof_check_err++;  in be_update_tx_err()
2682 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)  in lancer_update_tx_err()  argument
2686 		tx_stats(txo)->tx_tso_err++;  in lancer_update_tx_err()
2690 		tx_stats(txo)->tx_spoof_check_err++;  in lancer_update_tx_err()
2693 		tx_stats(txo)->tx_qinq_err++;  in lancer_update_tx_err()
2696 		tx_stats(txo)->tx_internal_parity_err++;  in lancer_update_tx_err()
2699 		tx_stats(txo)->tx_dma_err++;  in lancer_update_tx_err()
2702 		tx_stats(txo)->tx_sge_err++;  in lancer_update_tx_err()
2708 						struct be_tx_obj *txo)  in be_tx_compl_get()  argument
2710 	struct be_queue_info *tx_cq = &txo->cq;  in be_tx_compl_get()
2711 	struct be_tx_compl_info *txcp = &txo->txcp;  in be_tx_compl_get()
2726 			lancer_update_tx_err(txo, txcp->status);  in be_tx_compl_get()
2735 			be_update_tx_err(txo, txcp->status);  in be_tx_compl_get()
2748 			       struct be_tx_obj *txo, u16 last_index)  in be_tx_compl_process()  argument
2750 	struct sk_buff **sent_skbs = txo->sent_skb_list;  in be_tx_compl_process()
2751 	struct be_queue_info *txq = &txo->q;  in be_tx_compl_process()
2873 	struct be_tx_obj *txo;  in be_tx_compl_clean()  local
2880 		for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2883 			txq = &txo->q;  in be_tx_compl_clean()
2884 			while ((txcp = be_tx_compl_get(adapter, txo))) {  in be_tx_compl_clean()
2886 					be_tx_compl_process(adapter, txo,  in be_tx_compl_clean()
2891 				be_cq_notify(adapter, txo->cq.id, false, cmpl);  in be_tx_compl_clean()
2895 			if (!be_is_tx_compl_pending(txo))  in be_tx_compl_clean()
2907 	for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2908 		txq = &txo->q;  in be_tx_compl_clean()
2920 			num_wrbs = be_tx_compl_process(adapter, txo, end_idx);  in be_tx_compl_clean()
2923 			txo->pend_wrb_cnt = 0;  in be_tx_compl_clean()
3042 	struct be_tx_obj *txo;  in be_tx_queues_destroy()  local
3045 	for_all_tx_queues(adapter, txo, i) {  in be_tx_queues_destroy()
3046 		q = &txo->q;  in be_tx_queues_destroy()
3051 		q = &txo->cq;  in be_tx_queues_destroy()
3061 	struct be_tx_obj *txo;  in be_tx_qs_create()  local
3067 	for_all_tx_queues(adapter, txo, i) {  in be_tx_qs_create()
3068 		cq = &txo->cq;  in be_tx_qs_create()
3074 		u64_stats_init(&txo->stats.sync);  in be_tx_qs_create()
3075 		u64_stats_init(&txo->stats.sync_compl);  in be_tx_qs_create()
3085 		status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,  in be_tx_qs_create()
3090 		status = be_cmd_txq_create(adapter, txo);  in be_tx_qs_create()
3265 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_process_tx()  argument
3271 	while ((txcp = be_tx_compl_get(adapter, txo))) {  in be_process_tx()
3272 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);  in be_process_tx()
3277 		be_cq_notify(adapter, txo->cq.id, true, work_done);  in be_process_tx()
3278 		atomic_sub(num_wrbs, &txo->q.used);  in be_process_tx()
3283 		    be_can_txq_wake(txo)) {  in be_process_tx()
3287 		u64_stats_update_begin(&tx_stats(txo)->sync_compl);  in be_process_tx()
3288 		tx_stats(txo)->tx_compl += work_done;  in be_process_tx()
3289 		u64_stats_update_end(&tx_stats(txo)->sync_compl);  in be_process_tx()
3299 	struct be_tx_obj *txo;  in be_poll()  local
3304 	for_all_tx_queues_on_eq(adapter, eqo, txo, i)  in be_poll()
3305 		be_process_tx(adapter, txo, i);  in be_poll()
3797 	struct be_tx_obj *txo;  in be_open()  local
3816 	for_all_tx_queues(adapter, txo, i)  in be_open()
3817 		be_cq_notify(adapter, txo->cq.id, true, 0);  in be_open()