79 	for (txq_id = 0; txq_id < ARRAY_SIZE(trans->txqs.txq); txq_id++) {  in iwl_txq_gen2_tx_stop()
80 		if (!trans->txqs.txq[txq_id])  in iwl_txq_gen2_tx_stop()
90 					  struct iwl_txq *txq, u16 byte_cnt,  in iwl_pcie_gen2_update_byte_tbl()  argument
93 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_pcie_gen2_update_byte_tbl()
98 	if (WARN(idx >= txq->n_window, "%d >= %d\n", idx, txq->n_window))  in iwl_pcie_gen2_update_byte_tbl()
114 		struct iwl_gen3_bc_tbl *scd_bc_tbl_gen3 = txq->bc_tbl.addr;  in iwl_pcie_gen2_update_byte_tbl()
122 		struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr;  in iwl_pcie_gen2_update_byte_tbl()
136 void iwl_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_inc_wr_ptr()  argument
138 	lockdep_assert_held(&txq->lock);  in iwl_txq_inc_wr_ptr()
140 	IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr);  in iwl_txq_inc_wr_ptr()
146 	iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16));  in iwl_txq_inc_wr_ptr()
185 void iwl_txq_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_gen2_free_tfd()  argument
190 	int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr);  in iwl_txq_gen2_free_tfd()
192 	lockdep_assert_held(&txq->lock);  in iwl_txq_gen2_free_tfd()
194 	iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta,  in iwl_txq_gen2_free_tfd()
195 			       iwl_txq_get_tfd(trans, txq, idx));  in iwl_txq_gen2_free_tfd()
198 	if (txq->entries) {  in iwl_txq_gen2_free_tfd()
201 		skb = txq->entries[idx].skb;  in iwl_txq_gen2_free_tfd()
209 			txq->entries[idx].skb = NULL;  in iwl_txq_gen2_free_tfd()
530 					 struct iwl_txq *txq,  in iwl_txq_gen2_build_tx_amsdu()  argument
537 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tx_amsdu()
538 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tx_amsdu()
543 	tb_phys = iwl_txq_get_first_tb_dma(txq, idx);  in iwl_txq_gen2_build_tx_amsdu()
579 	memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE);  in iwl_txq_gen2_build_tx_amsdu()
617 				   struct iwl_txq *txq,  in iwl_txq_gen2_build_tx()  argument
625 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tx()
626 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tx()
632 	tb_phys = iwl_txq_get_first_tb_dma(txq, idx);  in iwl_txq_gen2_build_tx()
635 	memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE);  in iwl_txq_gen2_build_tx()
712 					   struct iwl_txq *txq,  in iwl_txq_gen2_build_tfd()  argument
718 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tfd()
719 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tfd()
745 		return iwl_txq_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb,  in iwl_txq_gen2_build_tfd()
747 	return iwl_txq_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta,  in iwl_txq_gen2_build_tfd()
784 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_tx()  local
802 	spin_lock(&txq->lock);  in iwl_txq_gen2_tx()
804 	if (iwl_txq_space(trans, txq) < txq->high_mark) {  in iwl_txq_gen2_tx()
805 		iwl_txq_stop(trans, txq);  in iwl_txq_gen2_tx()
808 		if (unlikely(iwl_txq_space(trans, txq) < 3)) {  in iwl_txq_gen2_tx()
815 			__skb_queue_tail(&txq->overflow_q, skb);  in iwl_txq_gen2_tx()
816 			spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
821 	idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_tx()
824 	txq->entries[idx].skb = skb;  in iwl_txq_gen2_tx()
825 	txq->entries[idx].cmd = dev_cmd;  in iwl_txq_gen2_tx()
832 	out_meta = &txq->entries[idx].meta;  in iwl_txq_gen2_tx()
835 	tfd = iwl_txq_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta);  in iwl_txq_gen2_tx()
837 		spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
854 	iwl_pcie_gen2_update_byte_tbl(trans, txq, cmd_len,  in iwl_txq_gen2_tx()
858 	if (txq->read_ptr == txq->write_ptr && txq->wd_timeout)  in iwl_txq_gen2_tx()
859 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);  in iwl_txq_gen2_tx()
862 	txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr);  in iwl_txq_gen2_tx()
863 	iwl_txq_inc_wr_ptr(trans, txq);  in iwl_txq_gen2_tx()
868 	spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
879 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_unmap()  local
881 	spin_lock_bh(&txq->lock);  in iwl_txq_gen2_unmap()
882 	while (txq->write_ptr != txq->read_ptr) {  in iwl_txq_gen2_unmap()
884 				   txq_id, txq->read_ptr);  in iwl_txq_gen2_unmap()
887 			int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr);  in iwl_txq_gen2_unmap()
888 			struct sk_buff *skb = txq->entries[idx].skb;  in iwl_txq_gen2_unmap()
895 		iwl_txq_gen2_free_tfd(trans, txq);  in iwl_txq_gen2_unmap()
896 		txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);  in iwl_txq_gen2_unmap()
899 	while (!skb_queue_empty(&txq->overflow_q)) {  in iwl_txq_gen2_unmap()
900 		struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);  in iwl_txq_gen2_unmap()
905 	spin_unlock_bh(&txq->lock);  in iwl_txq_gen2_unmap()
908 	iwl_wake_queue(trans, txq);  in iwl_txq_gen2_unmap()
912 				     struct iwl_txq *txq)  in iwl_txq_gen2_free_memory()  argument
917 	if (txq->tfds) {  in iwl_txq_gen2_free_memory()
919 				  trans->txqs.tfd.size * txq->n_window,  in iwl_txq_gen2_free_memory()
920 				  txq->tfds, txq->dma_addr);  in iwl_txq_gen2_free_memory()
922 				  sizeof(*txq->first_tb_bufs) * txq->n_window,  in iwl_txq_gen2_free_memory()
923 				  txq->first_tb_bufs, txq->first_tb_dma);  in iwl_txq_gen2_free_memory()
926 	kfree(txq->entries);  in iwl_txq_gen2_free_memory()
927 	if (txq->bc_tbl.addr)  in iwl_txq_gen2_free_memory()
929 			      txq->bc_tbl.addr, txq->bc_tbl.dma);  in iwl_txq_gen2_free_memory()
930 	kfree(txq);  in iwl_txq_gen2_free_memory()
943 	struct iwl_txq *txq;  in iwl_txq_gen2_free()  local
950 	txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_free()
952 	if (WARN_ON(!txq))  in iwl_txq_gen2_free()
959 		for (i = 0; i < txq->n_window; i++) {  in iwl_txq_gen2_free()
960 			kfree_sensitive(txq->entries[i].cmd);  in iwl_txq_gen2_free()
961 			kfree_sensitive(txq->entries[i].free_buf);  in iwl_txq_gen2_free()
963 	del_timer_sync(&txq->stuck_timer);  in iwl_txq_gen2_free()
965 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_gen2_free()
967 	trans->txqs.txq[txq_id] = NULL;  in iwl_txq_gen2_free()
998 int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num,  in iwl_txq_init()  argument
1005 	txq->need_update = false;  in iwl_txq_init()
1015 	ret = iwl_queue_init(txq, slots_num);  in iwl_txq_init()
1019 	spin_lock_init(&txq->lock);  in iwl_txq_init()
1024 		lockdep_set_class(&txq->lock, &iwl_txq_cmd_queue_lock_class);  in iwl_txq_init()
1027 	__skb_queue_head_init(&txq->overflow_q);  in iwl_txq_init()
1050 void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_log_scd_error()  argument
1052 	u32 txq_id = txq->id;  in iwl_txq_log_scd_error()
1059 			txq->read_ptr, txq->write_ptr);  in iwl_txq_log_scd_error()
1071 		jiffies_to_msecs(txq->wd_timeout),  in iwl_txq_log_scd_error()
1072 		txq->read_ptr, txq->write_ptr,  in iwl_txq_log_scd_error()
1082 	struct iwl_txq *txq = from_timer(txq, t, stuck_timer);  in iwl_txq_stuck_timer()  local
1083 	struct iwl_trans *trans = txq->trans;  in iwl_txq_stuck_timer()
1085 	spin_lock(&txq->lock);  in iwl_txq_stuck_timer()
1087 	if (txq->read_ptr == txq->write_ptr) {  in iwl_txq_stuck_timer()
1088 		spin_unlock(&txq->lock);  in iwl_txq_stuck_timer()
1091 	spin_unlock(&txq->lock);  in iwl_txq_stuck_timer()
1093 	iwl_txq_log_scd_error(trans, txq);  in iwl_txq_stuck_timer()
1098 int iwl_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num,  in iwl_txq_alloc()  argument
1106 	if (WARN_ON(txq->entries || txq->tfds))  in iwl_txq_alloc()
1112 	timer_setup(&txq->stuck_timer, iwl_txq_stuck_timer, 0);  in iwl_txq_alloc()
1113 	txq->trans = trans;  in iwl_txq_alloc()
1115 	txq->n_window = slots_num;  in iwl_txq_alloc()
1117 	txq->entries = kcalloc(slots_num,  in iwl_txq_alloc()
1121 	if (!txq->entries)  in iwl_txq_alloc()
1126 			txq->entries[i].cmd =  in iwl_txq_alloc()
1129 			if (!txq->entries[i].cmd)  in iwl_txq_alloc()
1135 	txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,  in iwl_txq_alloc()
1136 				       &txq->dma_addr, GFP_KERNEL);  in iwl_txq_alloc()
1137 	if (!txq->tfds)  in iwl_txq_alloc()
1140 	BUILD_BUG_ON(sizeof(*txq->first_tb_bufs) != IWL_FIRST_TB_SIZE_ALIGN);  in iwl_txq_alloc()
1142 	tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num;  in iwl_txq_alloc()
1144 	txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz,  in iwl_txq_alloc()
1145 						&txq->first_tb_dma,  in iwl_txq_alloc()
1147 	if (!txq->first_tb_bufs)  in iwl_txq_alloc()
1152 	dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr);  in iwl_txq_alloc()
1154 	if (txq->entries && cmd_queue)  in iwl_txq_alloc()
1156 			kfree(txq->entries[i].cmd);  in iwl_txq_alloc()
1157 	kfree(txq->entries);  in iwl_txq_alloc()
1158 	txq->entries = NULL;  in iwl_txq_alloc()
1168 	struct iwl_txq *txq;  in iwl_txq_dyn_alloc_dma()  local
1179 	txq = kzalloc(sizeof(*txq), GFP_KERNEL);  in iwl_txq_dyn_alloc_dma()
1180 	if (!txq)  in iwl_txq_dyn_alloc_dma()
1183 	txq->bc_tbl.addr = dma_pool_alloc(trans->txqs.bc_pool, GFP_KERNEL,  in iwl_txq_dyn_alloc_dma()
1184 					  &txq->bc_tbl.dma);  in iwl_txq_dyn_alloc_dma()
1185 	if (!txq->bc_tbl.addr) {  in iwl_txq_dyn_alloc_dma()
1187 		kfree(txq);  in iwl_txq_dyn_alloc_dma()
1191 	ret = iwl_txq_alloc(trans, txq, size, false);  in iwl_txq_dyn_alloc_dma()
1196 	ret = iwl_txq_init(trans, txq, size, false);  in iwl_txq_dyn_alloc_dma()
1202 	txq->wd_timeout = msecs_to_jiffies(timeout);  in iwl_txq_dyn_alloc_dma()
1204 	*intxq = txq;  in iwl_txq_dyn_alloc_dma()
1208 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_dyn_alloc_dma()
1212 static int iwl_txq_alloc_response(struct iwl_trans *trans, struct iwl_txq *txq,  in iwl_txq_alloc_response()  argument
1229 	if (qid >= ARRAY_SIZE(trans->txqs.txq)) {  in iwl_txq_alloc_response()
1241 	txq->id = qid;  in iwl_txq_alloc_response()
1242 	trans->txqs.txq[qid] = txq;  in iwl_txq_alloc_response()
1246 	txq->read_ptr = wr_ptr;  in iwl_txq_alloc_response()
1247 	txq->write_ptr = wr_ptr;  in iwl_txq_alloc_response()
1256 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_alloc_response()
1263 	struct iwl_txq *txq = NULL;  in iwl_txq_dyn_alloc()  local
1277 	ret = iwl_txq_dyn_alloc_dma(trans, &txq, size, timeout);  in iwl_txq_dyn_alloc()
1281 	cmd.tfdq_addr = cpu_to_le64(txq->dma_addr);  in iwl_txq_dyn_alloc()
1282 	cmd.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma);  in iwl_txq_dyn_alloc()
1289 	return iwl_txq_alloc_response(trans, txq, &hcmd);  in iwl_txq_dyn_alloc()
1292 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_dyn_alloc()
1316 	iwl_txq_gen2_free_memory(trans, trans->txqs.txq[queue]);  in iwl_txq_dyn_free()
1318 	trans->txqs.txq[queue] = NULL;  in iwl_txq_dyn_free()
1330 	for (i = 0; i < ARRAY_SIZE(trans->txqs.txq); i++) {  in iwl_txq_gen2_tx_free()
1331 		if (!trans->txqs.txq[i])  in iwl_txq_gen2_tx_free()
1344 	if (!trans->txqs.txq[txq_id]) {  in iwl_txq_gen2_init()
1350 		trans->txqs.txq[txq_id] = queue;  in iwl_txq_gen2_init()
1357 		queue = trans->txqs.txq[txq_id];  in iwl_txq_gen2_init()
1366 	trans->txqs.txq[txq_id]->id = txq_id;  in iwl_txq_gen2_init()
1410 			    struct iwl_txq *txq, int index)  in iwl_txq_gen1_tfd_unmap()  argument
1413 	void *tfd = iwl_txq_get_tfd(trans, txq, index);  in iwl_txq_gen1_tfd_unmap()
1463 				      struct iwl_txq *txq, u16 byte_cnt,  in iwl_txq_gen1_update_byte_cnt_tbl()  argument
1467 	int write_ptr = txq->write_ptr;  in iwl_txq_gen1_update_byte_cnt_tbl()
1468 	int txq_id = txq->id;  in iwl_txq_gen1_update_byte_cnt_tbl()
1472 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd;  in iwl_txq_gen1_update_byte_cnt_tbl()
1507 				     struct iwl_txq *txq)  in iwl_txq_gen1_inval_byte_cnt_tbl()  argument
1510 	int txq_id = txq->id;  in iwl_txq_gen1_inval_byte_cnt_tbl()
1511 	int read_ptr = txq->read_ptr;  in iwl_txq_gen1_inval_byte_cnt_tbl()
1514 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[read_ptr].cmd;  in iwl_txq_gen1_inval_byte_cnt_tbl()