22 					  struct iwl_txq *txq, u16 byte_cnt,  in iwl_pcie_gen2_update_byte_tbl()  argument
25 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_pcie_gen2_update_byte_tbl()
30 	if (WARN(idx >= txq->n_window, "%d >= %d\n", idx, txq->n_window))  in iwl_pcie_gen2_update_byte_tbl()
46 		struct iwl_gen3_bc_tbl_entry *scd_bc_tbl_gen3 = txq->bc_tbl.addr;  in iwl_pcie_gen2_update_byte_tbl()
54 		struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr;  in iwl_pcie_gen2_update_byte_tbl()
68 void iwl_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_inc_wr_ptr()  argument
70 	lockdep_assert_held(&txq->lock);  in iwl_txq_inc_wr_ptr()
72 	IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr);  in iwl_txq_inc_wr_ptr()
78 	iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16));  in iwl_txq_inc_wr_ptr()
117 void iwl_txq_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_gen2_free_tfd()  argument
122 	int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr);  in iwl_txq_gen2_free_tfd()
125 	lockdep_assert_held(&txq->lock);  in iwl_txq_gen2_free_tfd()
127 	if (!txq->entries)  in iwl_txq_gen2_free_tfd()
130 	iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta,  in iwl_txq_gen2_free_tfd()
131 			       iwl_txq_get_tfd(trans, txq, idx));  in iwl_txq_gen2_free_tfd()
133 	skb = txq->entries[idx].skb;  in iwl_txq_gen2_free_tfd()
141 		txq->entries[idx].skb = NULL;  in iwl_txq_gen2_free_tfd()
456 					 struct iwl_txq *txq,  in iwl_txq_gen2_build_tx_amsdu()  argument
463 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tx_amsdu()
464 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tx_amsdu()
469 	tb_phys = iwl_txq_get_first_tb_dma(txq, idx);  in iwl_txq_gen2_build_tx_amsdu()
505 	memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE);  in iwl_txq_gen2_build_tx_amsdu()
543 				   struct iwl_txq *txq,  in iwl_txq_gen2_build_tx()  argument
551 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tx()
552 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tx()
558 	tb_phys = iwl_txq_get_first_tb_dma(txq, idx);  in iwl_txq_gen2_build_tx()
561 	memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE);  in iwl_txq_gen2_build_tx()
638 					   struct iwl_txq *txq,  in iwl_txq_gen2_build_tfd()  argument
644 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_build_tfd()
645 	struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx);  in iwl_txq_gen2_build_tfd()
671 		return iwl_txq_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb,  in iwl_txq_gen2_build_tfd()
673 	return iwl_txq_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta,  in iwl_txq_gen2_build_tfd()
710 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_tx()  local
728 	spin_lock(&txq->lock);  in iwl_txq_gen2_tx()
730 	if (iwl_txq_space(trans, txq) < txq->high_mark) {  in iwl_txq_gen2_tx()
731 		iwl_txq_stop(trans, txq);  in iwl_txq_gen2_tx()
734 		if (unlikely(iwl_txq_space(trans, txq) < 3)) {  in iwl_txq_gen2_tx()
741 			__skb_queue_tail(&txq->overflow_q, skb);  in iwl_txq_gen2_tx()
742 			spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
747 	idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);  in iwl_txq_gen2_tx()
750 	txq->entries[idx].skb = skb;  in iwl_txq_gen2_tx()
751 	txq->entries[idx].cmd = dev_cmd;  in iwl_txq_gen2_tx()
758 	out_meta = &txq->entries[idx].meta;  in iwl_txq_gen2_tx()
761 	tfd = iwl_txq_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta);  in iwl_txq_gen2_tx()
763 		spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
780 	iwl_pcie_gen2_update_byte_tbl(trans, txq, cmd_len,  in iwl_txq_gen2_tx()
784 	if (txq->read_ptr == txq->write_ptr && txq->wd_timeout)  in iwl_txq_gen2_tx()
785 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);  in iwl_txq_gen2_tx()
788 	txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr);  in iwl_txq_gen2_tx()
789 	iwl_txq_inc_wr_ptr(trans, txq);  in iwl_txq_gen2_tx()
794 	spin_unlock(&txq->lock);  in iwl_txq_gen2_tx()
805 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_unmap()  local
807 	spin_lock_bh(&txq->lock);  in iwl_txq_gen2_unmap()
808 	while (txq->write_ptr != txq->read_ptr) {  in iwl_txq_gen2_unmap()
810 				   txq_id, txq->read_ptr);  in iwl_txq_gen2_unmap()
813 			int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr);  in iwl_txq_gen2_unmap()
814 			struct sk_buff *skb = txq->entries[idx].skb;  in iwl_txq_gen2_unmap()
819 		iwl_txq_gen2_free_tfd(trans, txq);  in iwl_txq_gen2_unmap()
820 		txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);  in iwl_txq_gen2_unmap()
823 	while (!skb_queue_empty(&txq->overflow_q)) {  in iwl_txq_gen2_unmap()
824 		struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);  in iwl_txq_gen2_unmap()
829 	spin_unlock_bh(&txq->lock);  in iwl_txq_gen2_unmap()
832 	iwl_wake_queue(trans, txq);  in iwl_txq_gen2_unmap()
836 				     struct iwl_txq *txq)  in iwl_txq_gen2_free_memory()  argument
841 	if (txq->tfds) {  in iwl_txq_gen2_free_memory()
843 				  trans->txqs.tfd.size * txq->n_window,  in iwl_txq_gen2_free_memory()
844 				  txq->tfds, txq->dma_addr);  in iwl_txq_gen2_free_memory()
846 				  sizeof(*txq->first_tb_bufs) * txq->n_window,  in iwl_txq_gen2_free_memory()
847 				  txq->first_tb_bufs, txq->first_tb_dma);  in iwl_txq_gen2_free_memory()
850 	kfree(txq->entries);  in iwl_txq_gen2_free_memory()
851 	if (txq->bc_tbl.addr)  in iwl_txq_gen2_free_memory()
853 			      txq->bc_tbl.addr, txq->bc_tbl.dma);  in iwl_txq_gen2_free_memory()
854 	kfree(txq);  in iwl_txq_gen2_free_memory()
867 	struct iwl_txq *txq;  in iwl_txq_gen2_free()  local
874 	txq = trans->txqs.txq[txq_id];  in iwl_txq_gen2_free()
876 	if (WARN_ON(!txq))  in iwl_txq_gen2_free()
883 		for (i = 0; i < txq->n_window; i++) {  in iwl_txq_gen2_free()
884 			kfree_sensitive(txq->entries[i].cmd);  in iwl_txq_gen2_free()
885 			kfree_sensitive(txq->entries[i].free_buf);  in iwl_txq_gen2_free()
887 	del_timer_sync(&txq->stuck_timer);  in iwl_txq_gen2_free()
889 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_gen2_free()
891 	trans->txqs.txq[txq_id] = NULL;  in iwl_txq_gen2_free()
922 int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num,  in iwl_txq_init()  argument
929 	txq->need_update = false;  in iwl_txq_init()
939 	ret = iwl_queue_init(txq, slots_num);  in iwl_txq_init()
943 	spin_lock_init(&txq->lock);  in iwl_txq_init()
948 		lockdep_set_class(&txq->lock, &iwl_txq_cmd_queue_lock_class);  in iwl_txq_init()
951 	__skb_queue_head_init(&txq->overflow_q);  in iwl_txq_init()
974 void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_log_scd_error()  argument
976 	u32 txq_id = txq->id;  in iwl_txq_log_scd_error()
983 			txq->read_ptr, txq->write_ptr);  in iwl_txq_log_scd_error()
995 		jiffies_to_msecs(txq->wd_timeout),  in iwl_txq_log_scd_error()
996 		txq->read_ptr, txq->write_ptr,  in iwl_txq_log_scd_error()
1006 	struct iwl_txq *txq = from_timer(txq, t, stuck_timer);  in iwl_txq_stuck_timer()  local
1007 	struct iwl_trans *trans = txq->trans;  in iwl_txq_stuck_timer()
1009 	spin_lock(&txq->lock);  in iwl_txq_stuck_timer()
1011 	if (txq->read_ptr == txq->write_ptr) {  in iwl_txq_stuck_timer()
1012 		spin_unlock(&txq->lock);  in iwl_txq_stuck_timer()
1015 	spin_unlock(&txq->lock);  in iwl_txq_stuck_timer()
1017 	iwl_txq_log_scd_error(trans, txq);  in iwl_txq_stuck_timer()
1022 int iwl_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num,  in iwl_txq_alloc()  argument
1030 	if (WARN_ON(txq->entries || txq->tfds))  in iwl_txq_alloc()
1036 	timer_setup(&txq->stuck_timer, iwl_txq_stuck_timer, 0);  in iwl_txq_alloc()
1037 	txq->trans = trans;  in iwl_txq_alloc()
1039 	txq->n_window = slots_num;  in iwl_txq_alloc()
1041 	txq->entries = kcalloc(slots_num,  in iwl_txq_alloc()
1045 	if (!txq->entries)  in iwl_txq_alloc()
1050 			txq->entries[i].cmd =  in iwl_txq_alloc()
1053 			if (!txq->entries[i].cmd)  in iwl_txq_alloc()
1059 	txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,  in iwl_txq_alloc()
1060 				       &txq->dma_addr, GFP_KERNEL);  in iwl_txq_alloc()
1061 	if (!txq->tfds)  in iwl_txq_alloc()
1064 	BUILD_BUG_ON(sizeof(*txq->first_tb_bufs) != IWL_FIRST_TB_SIZE_ALIGN);  in iwl_txq_alloc()
1066 	tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num;  in iwl_txq_alloc()
1068 	txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz,  in iwl_txq_alloc()
1069 						&txq->first_tb_dma,  in iwl_txq_alloc()
1071 	if (!txq->first_tb_bufs)  in iwl_txq_alloc()
1076 	dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr);  in iwl_txq_alloc()
1077 	txq->tfds = NULL;  in iwl_txq_alloc()
1079 	if (txq->entries && cmd_queue)  in iwl_txq_alloc()
1081 			kfree(txq->entries[i].cmd);  in iwl_txq_alloc()
1082 	kfree(txq->entries);  in iwl_txq_alloc()
1083 	txq->entries = NULL;  in iwl_txq_alloc()
1092 	struct iwl_txq *txq;  in iwl_txq_dyn_alloc_dma()  local
1103 	txq = kzalloc(sizeof(*txq), GFP_KERNEL);  in iwl_txq_dyn_alloc_dma()
1104 	if (!txq)  in iwl_txq_dyn_alloc_dma()
1107 	txq->bc_tbl.addr = dma_pool_alloc(trans->txqs.bc_pool, GFP_KERNEL,  in iwl_txq_dyn_alloc_dma()
1108 					  &txq->bc_tbl.dma);  in iwl_txq_dyn_alloc_dma()
1109 	if (!txq->bc_tbl.addr) {  in iwl_txq_dyn_alloc_dma()
1111 		kfree(txq);  in iwl_txq_dyn_alloc_dma()
1115 	ret = iwl_txq_alloc(trans, txq, size, false);  in iwl_txq_dyn_alloc_dma()
1120 	ret = iwl_txq_init(trans, txq, size, false);  in iwl_txq_dyn_alloc_dma()
1126 	txq->wd_timeout = msecs_to_jiffies(timeout);  in iwl_txq_dyn_alloc_dma()
1128 	return txq;  in iwl_txq_dyn_alloc_dma()
1131 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_dyn_alloc_dma()
1135 static int iwl_txq_alloc_response(struct iwl_trans *trans, struct iwl_txq *txq,  in iwl_txq_alloc_response()  argument
1152 	if (qid >= ARRAY_SIZE(trans->txqs.txq)) {  in iwl_txq_alloc_response()
1164 	if (WARN_ONCE(trans->txqs.txq[qid],  in iwl_txq_alloc_response()
1170 	txq->id = qid;  in iwl_txq_alloc_response()
1171 	trans->txqs.txq[qid] = txq;  in iwl_txq_alloc_response()
1175 	txq->read_ptr = wr_ptr;  in iwl_txq_alloc_response()
1176 	txq->write_ptr = wr_ptr;  in iwl_txq_alloc_response()
1185 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_alloc_response()
1192 	struct iwl_txq *txq;  in iwl_txq_dyn_alloc()  local
1206 	txq = iwl_txq_dyn_alloc_dma(trans, size, timeout);  in iwl_txq_dyn_alloc()
1207 	if (IS_ERR(txq))  in iwl_txq_dyn_alloc()
1208 		return PTR_ERR(txq);  in iwl_txq_dyn_alloc()
1212 		cmd.old.tfdq_addr = cpu_to_le64(txq->dma_addr);  in iwl_txq_dyn_alloc()
1213 		cmd.old.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma);  in iwl_txq_dyn_alloc()
1230 		cmd.new.u.add.tfdq_dram_addr = cpu_to_le64(txq->dma_addr);  in iwl_txq_dyn_alloc()
1231 		cmd.new.u.add.bc_dram_addr = cpu_to_le64(txq->bc_tbl.dma);  in iwl_txq_dyn_alloc()
1249 	return iwl_txq_alloc_response(trans, txq, &hcmd);  in iwl_txq_dyn_alloc()
1252 	iwl_txq_gen2_free_memory(trans, txq);  in iwl_txq_dyn_alloc()
1286 	for (i = 0; i < ARRAY_SIZE(trans->txqs.txq); i++) {  in iwl_txq_gen2_tx_free()
1287 		if (!trans->txqs.txq[i])  in iwl_txq_gen2_tx_free()
1300 	if (!trans->txqs.txq[txq_id]) {  in iwl_txq_gen2_init()
1306 		trans->txqs.txq[txq_id] = queue;  in iwl_txq_gen2_init()
1313 		queue = trans->txqs.txq[txq_id];  in iwl_txq_gen2_init()
1322 	trans->txqs.txq[txq_id]->id = txq_id;  in iwl_txq_gen2_init()
1366 			    struct iwl_txq *txq, int index)  in iwl_txq_gen1_tfd_unmap()  argument
1369 	void *tfd = iwl_txq_get_tfd(trans, txq, index);  in iwl_txq_gen1_tfd_unmap()
1419 				      struct iwl_txq *txq, u16 byte_cnt,  in iwl_txq_gen1_update_byte_cnt_tbl()  argument
1423 	int write_ptr = txq->write_ptr;  in iwl_txq_gen1_update_byte_cnt_tbl()
1424 	int txq_id = txq->id;  in iwl_txq_gen1_update_byte_cnt_tbl()
1428 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd;  in iwl_txq_gen1_update_byte_cnt_tbl()
1463 				     struct iwl_txq *txq)  in iwl_txq_gen1_inval_byte_cnt_tbl()  argument
1466 	int txq_id = txq->id;  in iwl_txq_gen1_inval_byte_cnt_tbl()
1467 	int read_ptr = txq->read_ptr;  in iwl_txq_gen1_inval_byte_cnt_tbl()
1470 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[read_ptr].cmd;  in iwl_txq_gen1_inval_byte_cnt_tbl()
1496 void iwl_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_free_tfd()  argument
1501 	int rd_ptr = txq->read_ptr;  in iwl_txq_free_tfd()
1502 	int idx = iwl_txq_get_cmd_index(txq, rd_ptr);  in iwl_txq_free_tfd()
1505 	lockdep_assert_held(&txq->lock);  in iwl_txq_free_tfd()
1507 	if (!txq->entries)  in iwl_txq_free_tfd()
1513 	iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr);  in iwl_txq_free_tfd()
1516 	skb = txq->entries[idx].skb;  in iwl_txq_free_tfd()
1524 		txq->entries[idx].skb = NULL;  in iwl_txq_free_tfd()
1528 void iwl_txq_progress(struct iwl_txq *txq)  in iwl_txq_progress()  argument
1530 	lockdep_assert_held(&txq->lock);  in iwl_txq_progress()
1532 	if (!txq->wd_timeout)  in iwl_txq_progress()
1539 	if (txq->frozen)  in iwl_txq_progress()
1546 	if (txq->read_ptr == txq->write_ptr)  in iwl_txq_progress()
1547 		del_timer(&txq->stuck_timer);  in iwl_txq_progress()
1549 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);  in iwl_txq_progress()
1556 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_reclaim()  local
1557 	int tfd_num = iwl_txq_get_cmd_index(txq, ssn);  in iwl_txq_reclaim()
1558 	int read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr);  in iwl_txq_reclaim()
1565 	spin_lock_bh(&txq->lock);  in iwl_txq_reclaim()
1577 			   txq_id, txq->read_ptr, tfd_num, ssn);  in iwl_txq_reclaim()
1583 	if (!iwl_txq_used(txq, last_to_free)) {  in iwl_txq_reclaim()
1588 			txq->write_ptr, txq->read_ptr);  in iwl_txq_reclaim()
1601 	     txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr),  in iwl_txq_reclaim()
1602 	     read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr)) {  in iwl_txq_reclaim()
1603 		struct sk_buff *skb = txq->entries[read_ptr].skb;  in iwl_txq_reclaim()
1612 		txq->entries[read_ptr].skb = NULL;  in iwl_txq_reclaim()
1615 			iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq);  in iwl_txq_reclaim()
1617 		iwl_txq_free_tfd(trans, txq);  in iwl_txq_reclaim()
1620 	iwl_txq_progress(txq);  in iwl_txq_reclaim()
1622 	if (iwl_txq_space(trans, txq) > txq->low_mark &&  in iwl_txq_reclaim()
1627 		skb_queue_splice_init(&txq->overflow_q, &overflow_skbs);  in iwl_txq_reclaim()
1636 		txq->overflow_tx = true;  in iwl_txq_reclaim()
1645 		spin_unlock_bh(&txq->lock);  in iwl_txq_reclaim()
1662 		if (iwl_txq_space(trans, txq) > txq->low_mark)  in iwl_txq_reclaim()
1663 			iwl_wake_queue(trans, txq);  in iwl_txq_reclaim()
1665 		spin_lock_bh(&txq->lock);  in iwl_txq_reclaim()
1666 		txq->overflow_tx = false;  in iwl_txq_reclaim()
1670 	spin_unlock_bh(&txq->lock);  in iwl_txq_reclaim()
1676 	struct iwl_txq *txq = trans->txqs.txq[txq_id];  in iwl_txq_set_q_ptrs()  local
1678 	spin_lock_bh(&txq->lock);  in iwl_txq_set_q_ptrs()
1680 	txq->write_ptr = ptr;  in iwl_txq_set_q_ptrs()
1681 	txq->read_ptr = txq->write_ptr;  in iwl_txq_set_q_ptrs()
1683 	spin_unlock_bh(&txq->lock);  in iwl_txq_set_q_ptrs()
1692 		struct iwl_txq *txq = trans->txqs.txq[queue];  in iwl_trans_txq_freeze_timer()  local
1695 		spin_lock_bh(&txq->lock);  in iwl_trans_txq_freeze_timer()
1699 		if (txq->frozen == freeze)  in iwl_trans_txq_freeze_timer()
1705 		txq->frozen = freeze;  in iwl_trans_txq_freeze_timer()
1707 		if (txq->read_ptr == txq->write_ptr)  in iwl_trans_txq_freeze_timer()
1712 						txq->stuck_timer.expires))) {  in iwl_trans_txq_freeze_timer()
1720 			txq->frozen_expiry_remainder =  in iwl_trans_txq_freeze_timer()
1721 				txq->stuck_timer.expires - now;  in iwl_trans_txq_freeze_timer()
1722 			del_timer(&txq->stuck_timer);  in iwl_trans_txq_freeze_timer()
1730 		mod_timer(&txq->stuck_timer,  in iwl_trans_txq_freeze_timer()
1731 			  now + txq->frozen_expiry_remainder);  in iwl_trans_txq_freeze_timer()
1734 		spin_unlock_bh(&txq->lock);  in iwl_trans_txq_freeze_timer()
1744 	struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id];  in iwl_trans_txq_send_hcmd_sync()  local
1775 			txq->read_ptr, txq->write_ptr);  in iwl_trans_txq_send_hcmd_sync()
1819 		txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB;  in iwl_trans_txq_send_hcmd_sync()