1 /******************************************************************************
2 *
3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
5 *
6 * GPL LICENSE SUMMARY
7 *
8 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
9 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
10 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
11 * Copyright(c) 2018 - 2020 Intel Corporation
12 *
13 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of version 2 of the GNU General Public License as
15 * published by the Free Software Foundation.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
20 * more details.
21 *
22 * The full GNU General Public License is included in this distribution in the
23 * file called COPYING.
24 *
25 * Contact Information:
26 * Intel Linux Wireless <linuxwifi@intel.com>
27 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
28 *
29 * BSD LICENSE
30 *
31 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
32 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
33 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
34 * Copyright(c) 2018 - 2020 Intel Corporation
35 * All rights reserved.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 *
41 * * Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * * Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in
45 * the documentation and/or other materials provided with the
46 * distribution.
47 * * Neither the name Intel Corporation nor the names of its
48 * contributors may be used to endorse or promote products derived
49 * from this software without specific prior written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
52 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
53 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
54 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
55 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
56 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
57 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
58 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
59 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
60 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
61 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
62 *
63 *****************************************************************************/
64 #include <linux/etherdevice.h>
65 #include <linux/ieee80211.h>
66 #include <linux/slab.h>
67 #include <linux/sched.h>
68 #include <net/ip6_checksum.h>
69 #include <net/tso.h>
70
71 #include "iwl-debug.h"
72 #include "iwl-csr.h"
73 #include "iwl-prph.h"
74 #include "iwl-io.h"
75 #include "iwl-scd.h"
76 #include "iwl-op-mode.h"
77 #include "internal.h"
78 #include "fw/api/tx.h"
79
80 /*************** DMA-QUEUE-GENERAL-FUNCTIONS *****
81 * DMA services
82 *
83 * Theory of operation
84 *
85 * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer
86 * of buffer descriptors, each of which points to one or more data buffers for
87 * the device to read from or fill. Driver and device exchange status of each
88 * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty
89 * entries in each circular buffer, to protect against confusing empty and full
90 * queue states.
91 *
92 * The device reads or writes the data in the queues via the device's several
93 * DMA/FIFO channels. Each queue is mapped to a single DMA channel.
94 *
95 * For Tx queue, there are low mark and high mark limits. If, after queuing
96 * the packet for Tx, free space become < low mark, Tx queue stopped. When
97 * reclaiming packets (on 'tx done IRQ), if free space become > high mark,
98 * Tx queue resumed.
99 *
100 ***************************************************/
101
102
iwl_pcie_alloc_dma_ptr(struct iwl_trans * trans,struct iwl_dma_ptr * ptr,size_t size)103 int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
104 struct iwl_dma_ptr *ptr, size_t size)
105 {
106 if (WARN_ON(ptr->addr))
107 return -EINVAL;
108
109 ptr->addr = dma_alloc_coherent(trans->dev, size,
110 &ptr->dma, GFP_KERNEL);
111 if (!ptr->addr)
112 return -ENOMEM;
113 ptr->size = size;
114 return 0;
115 }
116
iwl_pcie_free_dma_ptr(struct iwl_trans * trans,struct iwl_dma_ptr * ptr)117 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr)
118 {
119 if (unlikely(!ptr->addr))
120 return;
121
122 dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma);
123 memset(ptr, 0, sizeof(*ptr));
124 }
125
126 /*
127 * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware
128 */
iwl_pcie_txq_inc_wr_ptr(struct iwl_trans * trans,struct iwl_txq * txq)129 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
130 struct iwl_txq *txq)
131 {
132 u32 reg = 0;
133 int txq_id = txq->id;
134
135 lockdep_assert_held(&txq->lock);
136
137 /*
138 * explicitly wake up the NIC if:
139 * 1. shadow registers aren't enabled
140 * 2. NIC is woken up for CMD regardless of shadow outside this function
141 * 3. there is a chance that the NIC is asleep
142 */
143 if (!trans->trans_cfg->base_params->shadow_reg_enable &&
144 txq_id != trans->txqs.cmd.q_id &&
145 test_bit(STATUS_TPOWER_PMI, &trans->status)) {
146 /*
147 * wake up nic if it's powered down ...
148 * uCode will wake up, and interrupt us again, so next
149 * time we'll skip this part.
150 */
151 reg = iwl_read32(trans, CSR_UCODE_DRV_GP1);
152
153 if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) {
154 IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n",
155 txq_id, reg);
156 iwl_set_bit(trans, CSR_GP_CNTRL,
157 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
158 txq->need_update = true;
159 return;
160 }
161 }
162
163 /*
164 * if not in power-save mode, uCode will never sleep when we're
165 * trying to tx (during RFKILL, we're not trying to tx).
166 */
167 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr);
168 if (!txq->block)
169 iwl_write32(trans, HBUS_TARG_WRPTR,
170 txq->write_ptr | (txq_id << 8));
171 }
172
iwl_pcie_txq_check_wrptrs(struct iwl_trans * trans)173 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans)
174 {
175 int i;
176
177 for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
178 struct iwl_txq *txq = trans->txqs.txq[i];
179
180 if (!test_bit(i, trans->txqs.queue_used))
181 continue;
182
183 spin_lock_bh(&txq->lock);
184 if (txq->need_update) {
185 iwl_pcie_txq_inc_wr_ptr(trans, txq);
186 txq->need_update = false;
187 }
188 spin_unlock_bh(&txq->lock);
189 }
190 }
191
iwl_pcie_tfd_set_tb(struct iwl_trans * trans,void * tfd,u8 idx,dma_addr_t addr,u16 len)192 static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd,
193 u8 idx, dma_addr_t addr, u16 len)
194 {
195 struct iwl_tfd *tfd_fh = (void *)tfd;
196 struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx];
197
198 u16 hi_n_len = len << 4;
199
200 put_unaligned_le32(addr, &tb->lo);
201 hi_n_len |= iwl_get_dma_hi_addr(addr);
202
203 tb->hi_n_len = cpu_to_le16(hi_n_len);
204
205 tfd_fh->num_tbs = idx + 1;
206 }
207
208 /*
209 * iwl_pcie_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr]
210 * @trans - transport private data
211 * @txq - tx queue
212 * @dma_dir - the direction of the DMA mapping
213 *
214 * Does NOT advance any TFD circular buffer read/write indexes
215 * Does NOT free the TFD itself (which is within circular buffer)
216 */
iwl_pcie_txq_free_tfd(struct iwl_trans * trans,struct iwl_txq * txq)217 void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
218 {
219 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and
220 * idx is bounded by n_window
221 */
222 int rd_ptr = txq->read_ptr;
223 int idx = iwl_txq_get_cmd_index(txq, rd_ptr);
224
225 lockdep_assert_held(&txq->lock);
226
227 /* We have only q->n_window txq->entries, but we use
228 * TFD_QUEUE_SIZE_MAX tfds
229 */
230 iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr);
231
232 /* free SKB */
233 if (txq->entries) {
234 struct sk_buff *skb;
235
236 skb = txq->entries[idx].skb;
237
238 /* Can be called from irqs-disabled context
239 * If skb is not NULL, it means that the whole queue is being
240 * freed and that the queue is not empty - free the skb
241 */
242 if (skb) {
243 iwl_op_mode_free_skb(trans->op_mode, skb);
244 txq->entries[idx].skb = NULL;
245 }
246 }
247 }
248
iwl_pcie_txq_build_tfd(struct iwl_trans * trans,struct iwl_txq * txq,dma_addr_t addr,u16 len,bool reset)249 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq,
250 dma_addr_t addr, u16 len, bool reset)
251 {
252 void *tfd;
253 u32 num_tbs;
254
255 tfd = txq->tfds + trans->txqs.tfd.size * txq->write_ptr;
256
257 if (reset)
258 memset(tfd, 0, trans->txqs.tfd.size);
259
260 num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd);
261
262 /* Each TFD can point to a maximum max_tbs Tx buffers */
263 if (num_tbs >= trans->txqs.tfd.max_tbs) {
264 IWL_ERR(trans, "Error can not send more than %d chunks\n",
265 trans->txqs.tfd.max_tbs);
266 return -EINVAL;
267 }
268
269 if (WARN(addr & ~IWL_TX_DMA_MASK,
270 "Unaligned address = %llx\n", (unsigned long long)addr))
271 return -EINVAL;
272
273 iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len);
274
275 return num_tbs;
276 }
277
iwl_pcie_clear_cmd_in_flight(struct iwl_trans * trans)278 static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans)
279 {
280 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
281
282 lockdep_assert_held(&trans_pcie->reg_lock);
283
284 if (!trans->trans_cfg->base_params->apmg_wake_up_wa)
285 return;
286 if (WARN_ON(!trans_pcie->cmd_hold_nic_awake))
287 return;
288
289 trans_pcie->cmd_hold_nic_awake = false;
290 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
291 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
292 }
293
294 /*
295 * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's
296 */
iwl_pcie_txq_unmap(struct iwl_trans * trans,int txq_id)297 static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
298 {
299 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
300 struct iwl_txq *txq = trans->txqs.txq[txq_id];
301
302 spin_lock_bh(&txq->lock);
303 while (txq->write_ptr != txq->read_ptr) {
304 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
305 txq_id, txq->read_ptr);
306
307 if (txq_id != trans->txqs.cmd.q_id) {
308 struct sk_buff *skb = txq->entries[txq->read_ptr].skb;
309
310 if (WARN_ON_ONCE(!skb))
311 continue;
312
313 iwl_txq_free_tso_page(trans, skb);
314 }
315 iwl_pcie_txq_free_tfd(trans, txq);
316 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);
317
318 if (txq->read_ptr == txq->write_ptr) {
319 unsigned long flags;
320
321 spin_lock_irqsave(&trans_pcie->reg_lock, flags);
322 if (txq_id == trans->txqs.cmd.q_id)
323 iwl_pcie_clear_cmd_in_flight(trans);
324 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
325 }
326 }
327
328 while (!skb_queue_empty(&txq->overflow_q)) {
329 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);
330
331 iwl_op_mode_free_skb(trans->op_mode, skb);
332 }
333
334 spin_unlock_bh(&txq->lock);
335
336 /* just in case - this queue may have been stopped */
337 iwl_wake_queue(trans, txq);
338 }
339
340 /*
341 * iwl_pcie_txq_free - Deallocate DMA queue.
342 * @txq: Transmit queue to deallocate.
343 *
344 * Empty queue by removing and destroying all BD's.
345 * Free all buffers.
346 * 0-fill, but do not free "txq" descriptor structure.
347 */
iwl_pcie_txq_free(struct iwl_trans * trans,int txq_id)348 static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
349 {
350 struct iwl_txq *txq = trans->txqs.txq[txq_id];
351 struct device *dev = trans->dev;
352 int i;
353
354 if (WARN_ON(!txq))
355 return;
356
357 iwl_pcie_txq_unmap(trans, txq_id);
358
359 /* De-alloc array of command/tx buffers */
360 if (txq_id == trans->txqs.cmd.q_id)
361 for (i = 0; i < txq->n_window; i++) {
362 kfree_sensitive(txq->entries[i].cmd);
363 kfree_sensitive(txq->entries[i].free_buf);
364 }
365
366 /* De-alloc circular buffer of TFDs */
367 if (txq->tfds) {
368 dma_free_coherent(dev,
369 trans->txqs.tfd.size *
370 trans->trans_cfg->base_params->max_tfd_queue_size,
371 txq->tfds, txq->dma_addr);
372 txq->dma_addr = 0;
373 txq->tfds = NULL;
374
375 dma_free_coherent(dev,
376 sizeof(*txq->first_tb_bufs) * txq->n_window,
377 txq->first_tb_bufs, txq->first_tb_dma);
378 }
379
380 kfree(txq->entries);
381 txq->entries = NULL;
382
383 del_timer_sync(&txq->stuck_timer);
384
385 /* 0-fill queue descriptor structure */
386 memset(txq, 0, sizeof(*txq));
387 }
388
iwl_pcie_tx_start(struct iwl_trans * trans,u32 scd_base_addr)389 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
390 {
391 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
392 int nq = trans->trans_cfg->base_params->num_of_queues;
393 int chan;
394 u32 reg_val;
395 int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) -
396 SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32);
397
398 /* make sure all queue are not stopped/used */
399 memset(trans->txqs.queue_stopped, 0,
400 sizeof(trans->txqs.queue_stopped));
401 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used));
402
403 trans_pcie->scd_base_addr =
404 iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
405
406 WARN_ON(scd_base_addr != 0 &&
407 scd_base_addr != trans_pcie->scd_base_addr);
408
409 /* reset context data, TX status and translation data */
410 iwl_trans_write_mem(trans, trans_pcie->scd_base_addr +
411 SCD_CONTEXT_MEM_LOWER_BOUND,
412 NULL, clear_dwords);
413
414 iwl_write_prph(trans, SCD_DRAM_BASE_ADDR,
415 trans->txqs.scd_bc_tbls.dma >> 10);
416
417 /* The chain extension of the SCD doesn't work well. This feature is
418 * enabled by default by the HW, so we need to disable it manually.
419 */
420 if (trans->trans_cfg->base_params->scd_chain_ext_wa)
421 iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);
422
423 iwl_trans_ac_txq_enable(trans, trans->txqs.cmd.q_id,
424 trans->txqs.cmd.fifo,
425 trans->txqs.cmd.wdg_timeout);
426
427 /* Activate all Tx DMA/FIFO channels */
428 iwl_scd_activate_fifos(trans);
429
430 /* Enable DMA channel */
431 for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++)
432 iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan),
433 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
434 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
435
436 /* Update FH chicken bits */
437 reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG);
438 iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG,
439 reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
440
441 /* Enable L1-Active */
442 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
443 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
444 APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
445 }
446
iwl_trans_pcie_tx_reset(struct iwl_trans * trans)447 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans)
448 {
449 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
450 int txq_id;
451
452 /*
453 * we should never get here in gen2 trans mode return early to avoid
454 * having invalid accesses
455 */
456 if (WARN_ON_ONCE(trans->trans_cfg->gen2))
457 return;
458
459 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
460 txq_id++) {
461 struct iwl_txq *txq = trans->txqs.txq[txq_id];
462 if (trans->trans_cfg->use_tfh)
463 iwl_write_direct64(trans,
464 FH_MEM_CBBC_QUEUE(trans, txq_id),
465 txq->dma_addr);
466 else
467 iwl_write_direct32(trans,
468 FH_MEM_CBBC_QUEUE(trans, txq_id),
469 txq->dma_addr >> 8);
470 iwl_pcie_txq_unmap(trans, txq_id);
471 txq->read_ptr = 0;
472 txq->write_ptr = 0;
473 }
474
475 /* Tell NIC where to find the "keep warm" buffer */
476 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
477 trans_pcie->kw.dma >> 4);
478
479 /*
480 * Send 0 as the scd_base_addr since the device may have be reset
481 * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will
482 * contain garbage.
483 */
484 iwl_pcie_tx_start(trans, 0);
485 }
486
iwl_pcie_tx_stop_fh(struct iwl_trans * trans)487 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans)
488 {
489 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
490 unsigned long flags;
491 int ch, ret;
492 u32 mask = 0;
493
494 spin_lock(&trans_pcie->irq_lock);
495
496 if (!iwl_trans_grab_nic_access(trans, &flags))
497 goto out;
498
499 /* Stop each Tx DMA channel */
500 for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) {
501 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0);
502 mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch);
503 }
504
505 /* Wait for DMA channels to be idle */
506 ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000);
507 if (ret < 0)
508 IWL_ERR(trans,
509 "Failing on timeout while stopping DMA channel %d [0x%08x]\n",
510 ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG));
511
512 iwl_trans_release_nic_access(trans, &flags);
513
514 out:
515 spin_unlock(&trans_pcie->irq_lock);
516 }
517
518 /*
519 * iwl_pcie_tx_stop - Stop all Tx DMA channels
520 */
iwl_pcie_tx_stop(struct iwl_trans * trans)521 int iwl_pcie_tx_stop(struct iwl_trans *trans)
522 {
523 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
524 int txq_id;
525
526 /* Turn off all Tx DMA fifos */
527 iwl_scd_deactivate_fifos(trans);
528
529 /* Turn off all Tx DMA channels */
530 iwl_pcie_tx_stop_fh(trans);
531
532 /*
533 * This function can be called before the op_mode disabled the
534 * queues. This happens when we have an rfkill interrupt.
535 * Since we stop Tx altogether - mark the queues as stopped.
536 */
537 memset(trans->txqs.queue_stopped, 0,
538 sizeof(trans->txqs.queue_stopped));
539 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used));
540
541 /* This can happen: start_hw, stop_device */
542 if (!trans_pcie->txq_memory)
543 return 0;
544
545 /* Unmap DMA from host system and free skb's */
546 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
547 txq_id++)
548 iwl_pcie_txq_unmap(trans, txq_id);
549
550 return 0;
551 }
552
553 /*
554 * iwl_trans_tx_free - Free TXQ Context
555 *
556 * Destroy all TX DMA queues and structures
557 */
iwl_pcie_tx_free(struct iwl_trans * trans)558 void iwl_pcie_tx_free(struct iwl_trans *trans)
559 {
560 int txq_id;
561 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
562
563 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used));
564
565 /* Tx queues */
566 if (trans_pcie->txq_memory) {
567 for (txq_id = 0;
568 txq_id < trans->trans_cfg->base_params->num_of_queues;
569 txq_id++) {
570 iwl_pcie_txq_free(trans, txq_id);
571 trans->txqs.txq[txq_id] = NULL;
572 }
573 }
574
575 kfree(trans_pcie->txq_memory);
576 trans_pcie->txq_memory = NULL;
577
578 iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw);
579
580 iwl_pcie_free_dma_ptr(trans, &trans->txqs.scd_bc_tbls);
581 }
582
583 /*
584 * iwl_pcie_tx_alloc - allocate TX context
585 * Allocate all Tx DMA structures and initialize them
586 */
iwl_pcie_tx_alloc(struct iwl_trans * trans)587 static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
588 {
589 int ret;
590 int txq_id, slots_num;
591 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
592 u16 bc_tbls_size = trans->trans_cfg->base_params->num_of_queues;
593
594 if (WARN_ON(trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))
595 return -EINVAL;
596
597 bc_tbls_size *= sizeof(struct iwlagn_scd_bc_tbl);
598
599 /*It is not allowed to alloc twice, so warn when this happens.
600 * We cannot rely on the previous allocation, so free and fail */
601 if (WARN_ON(trans_pcie->txq_memory)) {
602 ret = -EINVAL;
603 goto error;
604 }
605
606 ret = iwl_pcie_alloc_dma_ptr(trans, &trans->txqs.scd_bc_tbls,
607 bc_tbls_size);
608 if (ret) {
609 IWL_ERR(trans, "Scheduler BC Table allocation failed\n");
610 goto error;
611 }
612
613 /* Alloc keep-warm buffer */
614 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE);
615 if (ret) {
616 IWL_ERR(trans, "Keep Warm allocation failed\n");
617 goto error;
618 }
619
620 trans_pcie->txq_memory =
621 kcalloc(trans->trans_cfg->base_params->num_of_queues,
622 sizeof(struct iwl_txq), GFP_KERNEL);
623 if (!trans_pcie->txq_memory) {
624 IWL_ERR(trans, "Not enough memory for txq\n");
625 ret = -ENOMEM;
626 goto error;
627 }
628
629 /* Alloc and init all Tx queues, including the command queue (#4/#9) */
630 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
631 txq_id++) {
632 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id);
633
634 if (cmd_queue)
635 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE,
636 trans->cfg->min_txq_size);
637 else
638 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE,
639 trans->cfg->min_256_ba_txq_size);
640 trans->txqs.txq[txq_id] = &trans_pcie->txq_memory[txq_id];
641 ret = iwl_txq_alloc(trans, trans->txqs.txq[txq_id], slots_num,
642 cmd_queue);
643 if (ret) {
644 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
645 goto error;
646 }
647 trans->txqs.txq[txq_id]->id = txq_id;
648 }
649
650 return 0;
651
652 error:
653 iwl_pcie_tx_free(trans);
654
655 return ret;
656 }
657
iwl_pcie_tx_init(struct iwl_trans * trans)658 int iwl_pcie_tx_init(struct iwl_trans *trans)
659 {
660 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
661 int ret;
662 int txq_id, slots_num;
663 bool alloc = false;
664
665 if (!trans_pcie->txq_memory) {
666 ret = iwl_pcie_tx_alloc(trans);
667 if (ret)
668 goto error;
669 alloc = true;
670 }
671
672 spin_lock(&trans_pcie->irq_lock);
673
674 /* Turn off all Tx DMA fifos */
675 iwl_scd_deactivate_fifos(trans);
676
677 /* Tell NIC where to find the "keep warm" buffer */
678 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
679 trans_pcie->kw.dma >> 4);
680
681 spin_unlock(&trans_pcie->irq_lock);
682
683 /* Alloc and init all Tx queues, including the command queue (#4/#9) */
684 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
685 txq_id++) {
686 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id);
687
688 if (cmd_queue)
689 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE,
690 trans->cfg->min_txq_size);
691 else
692 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE,
693 trans->cfg->min_256_ba_txq_size);
694 ret = iwl_txq_init(trans, trans->txqs.txq[txq_id], slots_num,
695 cmd_queue);
696 if (ret) {
697 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
698 goto error;
699 }
700
701 /*
702 * Tell nic where to find circular buffer of TFDs for a
703 * given Tx queue, and enable the DMA channel used for that
704 * queue.
705 * Circular buffer (TFD queue in DRAM) physical base address
706 */
707 iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id),
708 trans->txqs.txq[txq_id]->dma_addr >> 8);
709 }
710
711 iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE);
712 if (trans->trans_cfg->base_params->num_of_queues > 20)
713 iwl_set_bits_prph(trans, SCD_GP_CTRL,
714 SCD_GP_CTRL_ENABLE_31_QUEUES);
715
716 return 0;
717 error:
718 /*Upon error, free only if we allocated something */
719 if (alloc)
720 iwl_pcie_tx_free(trans);
721 return ret;
722 }
723
iwl_pcie_txq_progress(struct iwl_txq * txq)724 static inline void iwl_pcie_txq_progress(struct iwl_txq *txq)
725 {
726 lockdep_assert_held(&txq->lock);
727
728 if (!txq->wd_timeout)
729 return;
730
731 /*
732 * station is asleep and we send data - that must
733 * be uAPSD or PS-Poll. Don't rearm the timer.
734 */
735 if (txq->frozen)
736 return;
737
738 /*
739 * if empty delete timer, otherwise move timer forward
740 * since we're making progress on this queue
741 */
742 if (txq->read_ptr == txq->write_ptr)
743 del_timer(&txq->stuck_timer);
744 else
745 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
746 }
747
748 /* Frees buffers until index _not_ inclusive */
iwl_trans_pcie_reclaim(struct iwl_trans * trans,int txq_id,int ssn,struct sk_buff_head * skbs)749 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
750 struct sk_buff_head *skbs)
751 {
752 struct iwl_txq *txq = trans->txqs.txq[txq_id];
753 int tfd_num = iwl_txq_get_cmd_index(txq, ssn);
754 int read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr);
755 int last_to_free;
756
757 /* This function is not meant to release cmd queue*/
758 if (WARN_ON(txq_id == trans->txqs.cmd.q_id))
759 return;
760
761 spin_lock_bh(&txq->lock);
762
763 if (!test_bit(txq_id, trans->txqs.queue_used)) {
764 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n",
765 txq_id, ssn);
766 goto out;
767 }
768
769 if (read_ptr == tfd_num)
770 goto out;
771
772 IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n",
773 txq_id, txq->read_ptr, tfd_num, ssn);
774
775 /*Since we free until index _not_ inclusive, the one before index is
776 * the last we will free. This one must be used */
777 last_to_free = iwl_txq_dec_wrap(trans, tfd_num);
778
779 if (!iwl_txq_used(txq, last_to_free)) {
780 IWL_ERR(trans,
781 "%s: Read index for txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n",
782 __func__, txq_id, last_to_free,
783 trans->trans_cfg->base_params->max_tfd_queue_size,
784 txq->write_ptr, txq->read_ptr);
785 goto out;
786 }
787
788 if (WARN_ON(!skb_queue_empty(skbs)))
789 goto out;
790
791 for (;
792 read_ptr != tfd_num;
793 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr),
794 read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr)) {
795 struct sk_buff *skb = txq->entries[read_ptr].skb;
796
797 if (WARN_ON_ONCE(!skb))
798 continue;
799
800 iwl_txq_free_tso_page(trans, skb);
801
802 __skb_queue_tail(skbs, skb);
803
804 txq->entries[read_ptr].skb = NULL;
805
806 if (!trans->trans_cfg->use_tfh)
807 iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq);
808
809 iwl_pcie_txq_free_tfd(trans, txq);
810 }
811
812 iwl_pcie_txq_progress(txq);
813
814 if (iwl_txq_space(trans, txq) > txq->low_mark &&
815 test_bit(txq_id, trans->txqs.queue_stopped)) {
816 struct sk_buff_head overflow_skbs;
817
818 __skb_queue_head_init(&overflow_skbs);
819 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs);
820
821 /*
822 * We are going to transmit from the overflow queue.
823 * Remember this state so that wait_for_txq_empty will know we
824 * are adding more packets to the TFD queue. It cannot rely on
825 * the state of &txq->overflow_q, as we just emptied it, but
826 * haven't TXed the content yet.
827 */
828 txq->overflow_tx = true;
829
830 /*
831 * This is tricky: we are in reclaim path which is non
832 * re-entrant, so noone will try to take the access the
833 * txq data from that path. We stopped tx, so we can't
834 * have tx as well. Bottom line, we can unlock and re-lock
835 * later.
836 */
837 spin_unlock_bh(&txq->lock);
838
839 while (!skb_queue_empty(&overflow_skbs)) {
840 struct sk_buff *skb = __skb_dequeue(&overflow_skbs);
841 struct iwl_device_tx_cmd *dev_cmd_ptr;
842
843 dev_cmd_ptr = *(void **)((u8 *)skb->cb +
844 trans->txqs.dev_cmd_offs);
845
846 /*
847 * Note that we can very well be overflowing again.
848 * In that case, iwl_txq_space will be small again
849 * and we won't wake mac80211's queue.
850 */
851 iwl_trans_tx(trans, skb, dev_cmd_ptr, txq_id);
852 }
853
854 if (iwl_txq_space(trans, txq) > txq->low_mark)
855 iwl_wake_queue(trans, txq);
856
857 spin_lock_bh(&txq->lock);
858 txq->overflow_tx = false;
859 }
860
861 out:
862 spin_unlock_bh(&txq->lock);
863 }
864
865 /* Set wr_ptr of specific device and txq */
iwl_trans_pcie_set_q_ptrs(struct iwl_trans * trans,int txq_id,int ptr)866 void iwl_trans_pcie_set_q_ptrs(struct iwl_trans *trans, int txq_id, int ptr)
867 {
868 struct iwl_txq *txq = trans->txqs.txq[txq_id];
869
870 spin_lock_bh(&txq->lock);
871
872 txq->write_ptr = ptr;
873 txq->read_ptr = txq->write_ptr;
874
875 spin_unlock_bh(&txq->lock);
876 }
877
iwl_pcie_set_cmd_in_flight(struct iwl_trans * trans,const struct iwl_host_cmd * cmd)878 static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
879 const struct iwl_host_cmd *cmd)
880 {
881 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
882 int ret;
883
884 lockdep_assert_held(&trans_pcie->reg_lock);
885
886 /* Make sure the NIC is still alive in the bus */
887 if (test_bit(STATUS_TRANS_DEAD, &trans->status))
888 return -ENODEV;
889
890 /*
891 * wake up the NIC to make sure that the firmware will see the host
892 * command - we will let the NIC sleep once all the host commands
893 * returned. This needs to be done only on NICs that have
894 * apmg_wake_up_wa set.
895 */
896 if (trans->trans_cfg->base_params->apmg_wake_up_wa &&
897 !trans_pcie->cmd_hold_nic_awake) {
898 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
899 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
900
901 ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
902 CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
903 (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
904 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP),
905 15000);
906 if (ret < 0) {
907 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
908 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
909 IWL_ERR(trans, "Failed to wake NIC for hcmd\n");
910 return -EIO;
911 }
912 trans_pcie->cmd_hold_nic_awake = true;
913 }
914
915 return 0;
916 }
917
918 /*
919 * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd
920 *
921 * When FW advances 'R' index, all entries between old and new 'R' index
922 * need to be reclaimed. As result, some free space forms. If there is
923 * enough free space (> low mark), wake the stack that feeds us.
924 */
iwl_pcie_cmdq_reclaim(struct iwl_trans * trans,int txq_id,int idx)925 static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
926 {
927 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
928 struct iwl_txq *txq = trans->txqs.txq[txq_id];
929 unsigned long flags;
930 int nfreed = 0;
931 u16 r;
932
933 lockdep_assert_held(&txq->lock);
934
935 idx = iwl_txq_get_cmd_index(txq, idx);
936 r = iwl_txq_get_cmd_index(txq, txq->read_ptr);
937
938 if (idx >= trans->trans_cfg->base_params->max_tfd_queue_size ||
939 (!iwl_txq_used(txq, idx))) {
940 WARN_ONCE(test_bit(txq_id, trans->txqs.queue_used),
941 "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n",
942 __func__, txq_id, idx,
943 trans->trans_cfg->base_params->max_tfd_queue_size,
944 txq->write_ptr, txq->read_ptr);
945 return;
946 }
947
948 for (idx = iwl_txq_inc_wrap(trans, idx); r != idx;
949 r = iwl_txq_inc_wrap(trans, r)) {
950 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);
951
952 if (nfreed++ > 0) {
953 IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n",
954 idx, txq->write_ptr, r);
955 iwl_force_nmi(trans);
956 }
957 }
958
959 if (txq->read_ptr == txq->write_ptr) {
960 spin_lock_irqsave(&trans_pcie->reg_lock, flags);
961 iwl_pcie_clear_cmd_in_flight(trans);
962 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
963 }
964
965 iwl_pcie_txq_progress(txq);
966 }
967
iwl_pcie_txq_set_ratid_map(struct iwl_trans * trans,u16 ra_tid,u16 txq_id)968 static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid,
969 u16 txq_id)
970 {
971 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
972 u32 tbl_dw_addr;
973 u32 tbl_dw;
974 u16 scd_q2ratid;
975
976 scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK;
977
978 tbl_dw_addr = trans_pcie->scd_base_addr +
979 SCD_TRANS_TBL_OFFSET_QUEUE(txq_id);
980
981 tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr);
982
983 if (txq_id & 0x1)
984 tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF);
985 else
986 tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000);
987
988 iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw);
989
990 return 0;
991 }
992
993 /* Receiver address (actually, Rx station's index into station table),
994 * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */
995 #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid))
996
iwl_trans_pcie_txq_enable(struct iwl_trans * trans,int txq_id,u16 ssn,const struct iwl_trans_txq_scd_cfg * cfg,unsigned int wdg_timeout)997 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
998 const struct iwl_trans_txq_scd_cfg *cfg,
999 unsigned int wdg_timeout)
1000 {
1001 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1002 struct iwl_txq *txq = trans->txqs.txq[txq_id];
1003 int fifo = -1;
1004 bool scd_bug = false;
1005
1006 if (test_and_set_bit(txq_id, trans->txqs.queue_used))
1007 WARN_ONCE(1, "queue %d already used - expect issues", txq_id);
1008
1009 txq->wd_timeout = msecs_to_jiffies(wdg_timeout);
1010
1011 if (cfg) {
1012 fifo = cfg->fifo;
1013
1014 /* Disable the scheduler prior configuring the cmd queue */
1015 if (txq_id == trans->txqs.cmd.q_id &&
1016 trans_pcie->scd_set_active)
1017 iwl_scd_enable_set_active(trans, 0);
1018
1019 /* Stop this Tx queue before configuring it */
1020 iwl_scd_txq_set_inactive(trans, txq_id);
1021
1022 /* Set this queue as a chain-building queue unless it is CMD */
1023 if (txq_id != trans->txqs.cmd.q_id)
1024 iwl_scd_txq_set_chain(trans, txq_id);
1025
1026 if (cfg->aggregate) {
1027 u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid);
1028
1029 /* Map receiver-address / traffic-ID to this queue */
1030 iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id);
1031
1032 /* enable aggregations for the queue */
1033 iwl_scd_txq_enable_agg(trans, txq_id);
1034 txq->ampdu = true;
1035 } else {
1036 /*
1037 * disable aggregations for the queue, this will also
1038 * make the ra_tid mapping configuration irrelevant
1039 * since it is now a non-AGG queue.
1040 */
1041 iwl_scd_txq_disable_agg(trans, txq_id);
1042
1043 ssn = txq->read_ptr;
1044 }
1045 } else {
1046 /*
1047 * If we need to move the SCD write pointer by steps of
1048 * 0x40, 0x80 or 0xc0, it gets stuck. Avoids this and let
1049 * the op_mode know by returning true later.
1050 * Do this only in case cfg is NULL since this trick can
1051 * be done only if we have DQA enabled which is true for mvm
1052 * only. And mvm never sets a cfg pointer.
1053 * This is really ugly, but this is the easiest way out for
1054 * this sad hardware issue.
1055 * This bug has been fixed on devices 9000 and up.
1056 */
1057 scd_bug = !trans->trans_cfg->mq_rx_supported &&
1058 !((ssn - txq->write_ptr) & 0x3f) &&
1059 (ssn != txq->write_ptr);
1060 if (scd_bug)
1061 ssn++;
1062 }
1063
1064 /* Place first TFD at index corresponding to start sequence number.
1065 * Assumes that ssn_idx is valid (!= 0xFFF) */
1066 txq->read_ptr = (ssn & 0xff);
1067 txq->write_ptr = (ssn & 0xff);
1068 iwl_write_direct32(trans, HBUS_TARG_WRPTR,
1069 (ssn & 0xff) | (txq_id << 8));
1070
1071 if (cfg) {
1072 u8 frame_limit = cfg->frame_limit;
1073
1074 iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn);
1075
1076 /* Set up Tx window size and frame limit for this queue */
1077 iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr +
1078 SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0);
1079 iwl_trans_write_mem32(trans,
1080 trans_pcie->scd_base_addr +
1081 SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32),
1082 SCD_QUEUE_CTX_REG2_VAL(WIN_SIZE, frame_limit) |
1083 SCD_QUEUE_CTX_REG2_VAL(FRAME_LIMIT, frame_limit));
1084
1085 /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */
1086 iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id),
1087 (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1088 (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) |
1089 (1 << SCD_QUEUE_STTS_REG_POS_WSL) |
1090 SCD_QUEUE_STTS_REG_MSK);
1091
1092 /* enable the scheduler for this queue (only) */
1093 if (txq_id == trans->txqs.cmd.q_id &&
1094 trans_pcie->scd_set_active)
1095 iwl_scd_enable_set_active(trans, BIT(txq_id));
1096
1097 IWL_DEBUG_TX_QUEUES(trans,
1098 "Activate queue %d on FIFO %d WrPtr: %d\n",
1099 txq_id, fifo, ssn & 0xff);
1100 } else {
1101 IWL_DEBUG_TX_QUEUES(trans,
1102 "Activate queue %d WrPtr: %d\n",
1103 txq_id, ssn & 0xff);
1104 }
1105
1106 return scd_bug;
1107 }
1108
iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans * trans,u32 txq_id,bool shared_mode)1109 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
1110 bool shared_mode)
1111 {
1112 struct iwl_txq *txq = trans->txqs.txq[txq_id];
1113
1114 txq->ampdu = !shared_mode;
1115 }
1116
iwl_trans_pcie_txq_disable(struct iwl_trans * trans,int txq_id,bool configure_scd)1117 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id,
1118 bool configure_scd)
1119 {
1120 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1121 u32 stts_addr = trans_pcie->scd_base_addr +
1122 SCD_TX_STTS_QUEUE_OFFSET(txq_id);
1123 static const u32 zero_val[4] = {};
1124
1125 trans->txqs.txq[txq_id]->frozen_expiry_remainder = 0;
1126 trans->txqs.txq[txq_id]->frozen = false;
1127
1128 /*
1129 * Upon HW Rfkill - we stop the device, and then stop the queues
1130 * in the op_mode. Just for the sake of the simplicity of the op_mode,
1131 * allow the op_mode to call txq_disable after it already called
1132 * stop_device.
1133 */
1134 if (!test_and_clear_bit(txq_id, trans->txqs.queue_used)) {
1135 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status),
1136 "queue %d not used", txq_id);
1137 return;
1138 }
1139
1140 if (configure_scd) {
1141 iwl_scd_txq_set_inactive(trans, txq_id);
1142
1143 iwl_trans_write_mem(trans, stts_addr, (void *)zero_val,
1144 ARRAY_SIZE(zero_val));
1145 }
1146
1147 iwl_pcie_txq_unmap(trans, txq_id);
1148 trans->txqs.txq[txq_id]->ampdu = false;
1149
1150 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id);
1151 }
1152
1153 /*************** HOST COMMAND QUEUE FUNCTIONS *****/
1154
1155 /*
1156 * iwl_pcie_enqueue_hcmd - enqueue a uCode command
1157 * @priv: device private data point
1158 * @cmd: a pointer to the ucode command structure
1159 *
1160 * The function returns < 0 values to indicate the operation
1161 * failed. On success, it returns the index (>= 0) of command in the
1162 * command queue.
1163 */
iwl_pcie_enqueue_hcmd(struct iwl_trans * trans,struct iwl_host_cmd * cmd)1164 static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
1165 struct iwl_host_cmd *cmd)
1166 {
1167 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1168 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id];
1169 struct iwl_device_cmd *out_cmd;
1170 struct iwl_cmd_meta *out_meta;
1171 unsigned long flags;
1172 void *dup_buf = NULL;
1173 dma_addr_t phys_addr;
1174 int idx;
1175 u16 copy_size, cmd_size, tb0_size;
1176 bool had_nocopy = false;
1177 u8 group_id = iwl_cmd_groupid(cmd->id);
1178 int i, ret;
1179 u32 cmd_pos;
1180 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD];
1181 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
1182
1183 if (WARN(!trans->wide_cmd_header &&
1184 group_id > IWL_ALWAYS_LONG_GROUP,
1185 "unsupported wide command %#x\n", cmd->id))
1186 return -EINVAL;
1187
1188 if (group_id != 0) {
1189 copy_size = sizeof(struct iwl_cmd_header_wide);
1190 cmd_size = sizeof(struct iwl_cmd_header_wide);
1191 } else {
1192 copy_size = sizeof(struct iwl_cmd_header);
1193 cmd_size = sizeof(struct iwl_cmd_header);
1194 }
1195
1196 /* need one for the header if the first is NOCOPY */
1197 BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1);
1198
1199 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1200 cmddata[i] = cmd->data[i];
1201 cmdlen[i] = cmd->len[i];
1202
1203 if (!cmd->len[i])
1204 continue;
1205
1206 /* need at least IWL_FIRST_TB_SIZE copied */
1207 if (copy_size < IWL_FIRST_TB_SIZE) {
1208 int copy = IWL_FIRST_TB_SIZE - copy_size;
1209
1210 if (copy > cmdlen[i])
1211 copy = cmdlen[i];
1212 cmdlen[i] -= copy;
1213 cmddata[i] += copy;
1214 copy_size += copy;
1215 }
1216
1217 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) {
1218 had_nocopy = true;
1219 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) {
1220 idx = -EINVAL;
1221 goto free_dup_buf;
1222 }
1223 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) {
1224 /*
1225 * This is also a chunk that isn't copied
1226 * to the static buffer so set had_nocopy.
1227 */
1228 had_nocopy = true;
1229
1230 /* only allowed once */
1231 if (WARN_ON(dup_buf)) {
1232 idx = -EINVAL;
1233 goto free_dup_buf;
1234 }
1235
1236 dup_buf = kmemdup(cmddata[i], cmdlen[i],
1237 GFP_ATOMIC);
1238 if (!dup_buf)
1239 return -ENOMEM;
1240 } else {
1241 /* NOCOPY must not be followed by normal! */
1242 if (WARN_ON(had_nocopy)) {
1243 idx = -EINVAL;
1244 goto free_dup_buf;
1245 }
1246 copy_size += cmdlen[i];
1247 }
1248 cmd_size += cmd->len[i];
1249 }
1250
1251 /*
1252 * If any of the command structures end up being larger than
1253 * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically
1254 * allocated into separate TFDs, then we will need to
1255 * increase the size of the buffers.
1256 */
1257 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE,
1258 "Command %s (%#x) is too large (%d bytes)\n",
1259 iwl_get_cmd_string(trans, cmd->id),
1260 cmd->id, copy_size)) {
1261 idx = -EINVAL;
1262 goto free_dup_buf;
1263 }
1264
1265 spin_lock_bh(&txq->lock);
1266
1267 if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
1268 spin_unlock_bh(&txq->lock);
1269
1270 IWL_ERR(trans, "No space in command queue\n");
1271 iwl_op_mode_cmd_queue_full(trans->op_mode);
1272 idx = -ENOSPC;
1273 goto free_dup_buf;
1274 }
1275
1276 idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);
1277 out_cmd = txq->entries[idx].cmd;
1278 out_meta = &txq->entries[idx].meta;
1279
1280 memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */
1281 if (cmd->flags & CMD_WANT_SKB)
1282 out_meta->source = cmd;
1283
1284 /* set up the header */
1285 if (group_id != 0) {
1286 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id);
1287 out_cmd->hdr_wide.group_id = group_id;
1288 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id);
1289 out_cmd->hdr_wide.length =
1290 cpu_to_le16(cmd_size -
1291 sizeof(struct iwl_cmd_header_wide));
1292 out_cmd->hdr_wide.reserved = 0;
1293 out_cmd->hdr_wide.sequence =
1294 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) |
1295 INDEX_TO_SEQ(txq->write_ptr));
1296
1297 cmd_pos = sizeof(struct iwl_cmd_header_wide);
1298 copy_size = sizeof(struct iwl_cmd_header_wide);
1299 } else {
1300 out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id);
1301 out_cmd->hdr.sequence =
1302 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) |
1303 INDEX_TO_SEQ(txq->write_ptr));
1304 out_cmd->hdr.group_id = 0;
1305
1306 cmd_pos = sizeof(struct iwl_cmd_header);
1307 copy_size = sizeof(struct iwl_cmd_header);
1308 }
1309
1310 /* and copy the data that needs to be copied */
1311 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1312 int copy;
1313
1314 if (!cmd->len[i])
1315 continue;
1316
1317 /* copy everything if not nocopy/dup */
1318 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
1319 IWL_HCMD_DFL_DUP))) {
1320 copy = cmd->len[i];
1321
1322 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
1323 cmd_pos += copy;
1324 copy_size += copy;
1325 continue;
1326 }
1327
1328 /*
1329 * Otherwise we need at least IWL_FIRST_TB_SIZE copied
1330 * in total (for bi-directional DMA), but copy up to what
1331 * we can fit into the payload for debug dump purposes.
1332 */
1333 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]);
1334
1335 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
1336 cmd_pos += copy;
1337
1338 /* However, treat copy_size the proper way, we need it below */
1339 if (copy_size < IWL_FIRST_TB_SIZE) {
1340 copy = IWL_FIRST_TB_SIZE - copy_size;
1341
1342 if (copy > cmd->len[i])
1343 copy = cmd->len[i];
1344 copy_size += copy;
1345 }
1346 }
1347
1348 IWL_DEBUG_HC(trans,
1349 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n",
1350 iwl_get_cmd_string(trans, cmd->id),
1351 group_id, out_cmd->hdr.cmd,
1352 le16_to_cpu(out_cmd->hdr.sequence),
1353 cmd_size, txq->write_ptr, idx, trans->txqs.cmd.q_id);
1354
1355 /* start the TFD with the minimum copy bytes */
1356 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
1357 memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size);
1358 iwl_pcie_txq_build_tfd(trans, txq,
1359 iwl_txq_get_first_tb_dma(txq, idx),
1360 tb0_size, true);
1361
1362 /* map first command fragment, if any remains */
1363 if (copy_size > tb0_size) {
1364 phys_addr = dma_map_single(trans->dev,
1365 ((u8 *)&out_cmd->hdr) + tb0_size,
1366 copy_size - tb0_size,
1367 DMA_TO_DEVICE);
1368 if (dma_mapping_error(trans->dev, phys_addr)) {
1369 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq,
1370 txq->write_ptr);
1371 idx = -ENOMEM;
1372 goto out;
1373 }
1374
1375 iwl_pcie_txq_build_tfd(trans, txq, phys_addr,
1376 copy_size - tb0_size, false);
1377 }
1378
1379 /* map the remaining (adjusted) nocopy/dup fragments */
1380 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1381 const void *data = cmddata[i];
1382
1383 if (!cmdlen[i])
1384 continue;
1385 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
1386 IWL_HCMD_DFL_DUP)))
1387 continue;
1388 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP)
1389 data = dup_buf;
1390 phys_addr = dma_map_single(trans->dev, (void *)data,
1391 cmdlen[i], DMA_TO_DEVICE);
1392 if (dma_mapping_error(trans->dev, phys_addr)) {
1393 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq,
1394 txq->write_ptr);
1395 idx = -ENOMEM;
1396 goto out;
1397 }
1398
1399 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false);
1400 }
1401
1402 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE);
1403 out_meta->flags = cmd->flags;
1404 if (WARN_ON_ONCE(txq->entries[idx].free_buf))
1405 kfree_sensitive(txq->entries[idx].free_buf);
1406 txq->entries[idx].free_buf = dup_buf;
1407
1408 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide);
1409
1410 /* start timer if queue currently empty */
1411 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout)
1412 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
1413
1414 spin_lock_irqsave(&trans_pcie->reg_lock, flags);
1415 ret = iwl_pcie_set_cmd_in_flight(trans, cmd);
1416 if (ret < 0) {
1417 idx = ret;
1418 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
1419 goto out;
1420 }
1421
1422 /* Increment and update queue's write index */
1423 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr);
1424 iwl_pcie_txq_inc_wr_ptr(trans, txq);
1425
1426 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
1427
1428 out:
1429 spin_unlock_bh(&txq->lock);
1430 free_dup_buf:
1431 if (idx < 0)
1432 kfree(dup_buf);
1433 return idx;
1434 }
1435
1436 /*
1437 * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them
1438 * @rxb: Rx buffer to reclaim
1439 */
iwl_pcie_hcmd_complete(struct iwl_trans * trans,struct iwl_rx_cmd_buffer * rxb)1440 void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
1441 struct iwl_rx_cmd_buffer *rxb)
1442 {
1443 struct iwl_rx_packet *pkt = rxb_addr(rxb);
1444 u16 sequence = le16_to_cpu(pkt->hdr.sequence);
1445 u8 group_id;
1446 u32 cmd_id;
1447 int txq_id = SEQ_TO_QUEUE(sequence);
1448 int index = SEQ_TO_INDEX(sequence);
1449 int cmd_index;
1450 struct iwl_device_cmd *cmd;
1451 struct iwl_cmd_meta *meta;
1452 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1453 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id];
1454
1455 /* If a Tx command is being handled and it isn't in the actual
1456 * command queue then there a command routing bug has been introduced
1457 * in the queue management code. */
1458 if (WARN(txq_id != trans->txqs.cmd.q_id,
1459 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n",
1460 txq_id, trans->txqs.cmd.q_id, sequence, txq->read_ptr,
1461 txq->write_ptr)) {
1462 iwl_print_hex_error(trans, pkt, 32);
1463 return;
1464 }
1465
1466 spin_lock_bh(&txq->lock);
1467
1468 cmd_index = iwl_txq_get_cmd_index(txq, index);
1469 cmd = txq->entries[cmd_index].cmd;
1470 meta = &txq->entries[cmd_index].meta;
1471 group_id = cmd->hdr.group_id;
1472 cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0);
1473
1474 iwl_txq_gen1_tfd_unmap(trans, meta, txq, index);
1475
1476 /* Input error checking is done when commands are added to queue. */
1477 if (meta->flags & CMD_WANT_SKB) {
1478 struct page *p = rxb_steal_page(rxb);
1479
1480 meta->source->resp_pkt = pkt;
1481 meta->source->_rx_page_addr = (unsigned long)page_address(p);
1482 meta->source->_rx_page_order = trans_pcie->rx_page_order;
1483 }
1484
1485 if (meta->flags & CMD_WANT_ASYNC_CALLBACK)
1486 iwl_op_mode_async_cb(trans->op_mode, cmd);
1487
1488 iwl_pcie_cmdq_reclaim(trans, txq_id, index);
1489
1490 if (!(meta->flags & CMD_ASYNC)) {
1491 if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) {
1492 IWL_WARN(trans,
1493 "HCMD_ACTIVE already clear for command %s\n",
1494 iwl_get_cmd_string(trans, cmd_id));
1495 }
1496 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1497 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
1498 iwl_get_cmd_string(trans, cmd_id));
1499 wake_up(&trans_pcie->wait_command_queue);
1500 }
1501
1502 meta->flags = 0;
1503
1504 spin_unlock_bh(&txq->lock);
1505 }
1506
1507 #define HOST_COMPLETE_TIMEOUT (2 * HZ)
1508
iwl_pcie_send_hcmd_async(struct iwl_trans * trans,struct iwl_host_cmd * cmd)1509 static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans,
1510 struct iwl_host_cmd *cmd)
1511 {
1512 int ret;
1513
1514 /* An asynchronous command can not expect an SKB to be set. */
1515 if (WARN_ON(cmd->flags & CMD_WANT_SKB))
1516 return -EINVAL;
1517
1518 ret = iwl_pcie_enqueue_hcmd(trans, cmd);
1519 if (ret < 0) {
1520 IWL_ERR(trans,
1521 "Error sending %s: enqueue_hcmd failed: %d\n",
1522 iwl_get_cmd_string(trans, cmd->id), ret);
1523 return ret;
1524 }
1525 return 0;
1526 }
1527
iwl_pcie_send_hcmd_sync(struct iwl_trans * trans,struct iwl_host_cmd * cmd)1528 static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans,
1529 struct iwl_host_cmd *cmd)
1530 {
1531 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1532 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id];
1533 int cmd_idx;
1534 int ret;
1535
1536 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n",
1537 iwl_get_cmd_string(trans, cmd->id));
1538
1539 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE,
1540 &trans->status),
1541 "Command %s: a command is already active!\n",
1542 iwl_get_cmd_string(trans, cmd->id)))
1543 return -EIO;
1544
1545 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n",
1546 iwl_get_cmd_string(trans, cmd->id));
1547
1548 cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd);
1549 if (cmd_idx < 0) {
1550 ret = cmd_idx;
1551 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1552 IWL_ERR(trans,
1553 "Error sending %s: enqueue_hcmd failed: %d\n",
1554 iwl_get_cmd_string(trans, cmd->id), ret);
1555 return ret;
1556 }
1557
1558 ret = wait_event_timeout(trans_pcie->wait_command_queue,
1559 !test_bit(STATUS_SYNC_HCMD_ACTIVE,
1560 &trans->status),
1561 HOST_COMPLETE_TIMEOUT);
1562 if (!ret) {
1563 IWL_ERR(trans, "Error sending %s: time out after %dms.\n",
1564 iwl_get_cmd_string(trans, cmd->id),
1565 jiffies_to_msecs(HOST_COMPLETE_TIMEOUT));
1566
1567 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n",
1568 txq->read_ptr, txq->write_ptr);
1569
1570 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1571 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
1572 iwl_get_cmd_string(trans, cmd->id));
1573 ret = -ETIMEDOUT;
1574
1575 iwl_trans_pcie_sync_nmi(trans);
1576 goto cancel;
1577 }
1578
1579 if (test_bit(STATUS_FW_ERROR, &trans->status)) {
1580 iwl_trans_pcie_dump_regs(trans);
1581 IWL_ERR(trans, "FW error in SYNC CMD %s\n",
1582 iwl_get_cmd_string(trans, cmd->id));
1583 dump_stack();
1584 ret = -EIO;
1585 goto cancel;
1586 }
1587
1588 if (!(cmd->flags & CMD_SEND_IN_RFKILL) &&
1589 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) {
1590 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n");
1591 ret = -ERFKILL;
1592 goto cancel;
1593 }
1594
1595 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) {
1596 IWL_ERR(trans, "Error: Response NULL in '%s'\n",
1597 iwl_get_cmd_string(trans, cmd->id));
1598 ret = -EIO;
1599 goto cancel;
1600 }
1601
1602 return 0;
1603
1604 cancel:
1605 if (cmd->flags & CMD_WANT_SKB) {
1606 /*
1607 * Cancel the CMD_WANT_SKB flag for the cmd in the
1608 * TX cmd queue. Otherwise in case the cmd comes
1609 * in later, it will possibly set an invalid
1610 * address (cmd->meta.source).
1611 */
1612 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB;
1613 }
1614
1615 if (cmd->resp_pkt) {
1616 iwl_free_resp(cmd);
1617 cmd->resp_pkt = NULL;
1618 }
1619
1620 return ret;
1621 }
1622
iwl_trans_pcie_send_hcmd(struct iwl_trans * trans,struct iwl_host_cmd * cmd)1623 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
1624 {
1625 /* Make sure the NIC is still alive in the bus */
1626 if (test_bit(STATUS_TRANS_DEAD, &trans->status))
1627 return -ENODEV;
1628
1629 if (!(cmd->flags & CMD_SEND_IN_RFKILL) &&
1630 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) {
1631 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n",
1632 cmd->id);
1633 return -ERFKILL;
1634 }
1635
1636 if (cmd->flags & CMD_ASYNC)
1637 return iwl_pcie_send_hcmd_async(trans, cmd);
1638
1639 /* We still can fail on RFKILL that can be asserted while we wait */
1640 return iwl_pcie_send_hcmd_sync(trans, cmd);
1641 }
1642
iwl_fill_data_tbs(struct iwl_trans * trans,struct sk_buff * skb,struct iwl_txq * txq,u8 hdr_len,struct iwl_cmd_meta * out_meta)1643 static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb,
1644 struct iwl_txq *txq, u8 hdr_len,
1645 struct iwl_cmd_meta *out_meta)
1646 {
1647 u16 head_tb_len;
1648 int i;
1649
1650 /*
1651 * Set up TFD's third entry to point directly to remainder
1652 * of skb's head, if any
1653 */
1654 head_tb_len = skb_headlen(skb) - hdr_len;
1655
1656 if (head_tb_len > 0) {
1657 dma_addr_t tb_phys = dma_map_single(trans->dev,
1658 skb->data + hdr_len,
1659 head_tb_len, DMA_TO_DEVICE);
1660 if (unlikely(dma_mapping_error(trans->dev, tb_phys)))
1661 return -EINVAL;
1662 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb->data + hdr_len,
1663 tb_phys, head_tb_len);
1664 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, head_tb_len, false);
1665 }
1666
1667 /* set up the remaining entries to point to the data */
1668 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1669 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1670 dma_addr_t tb_phys;
1671 int tb_idx;
1672
1673 if (!skb_frag_size(frag))
1674 continue;
1675
1676 tb_phys = skb_frag_dma_map(trans->dev, frag, 0,
1677 skb_frag_size(frag), DMA_TO_DEVICE);
1678
1679 if (unlikely(dma_mapping_error(trans->dev, tb_phys)))
1680 return -EINVAL;
1681 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag),
1682 tb_phys, skb_frag_size(frag));
1683 tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys,
1684 skb_frag_size(frag), false);
1685 if (tb_idx < 0)
1686 return tb_idx;
1687
1688 out_meta->tbs |= BIT(tb_idx);
1689 }
1690
1691 return 0;
1692 }
1693
1694 #ifdef CONFIG_INET
iwl_compute_pseudo_hdr_csum(void * iph,struct tcphdr * tcph,bool ipv6,unsigned int len)1695 static void iwl_compute_pseudo_hdr_csum(void *iph, struct tcphdr *tcph,
1696 bool ipv6, unsigned int len)
1697 {
1698 if (ipv6) {
1699 struct ipv6hdr *iphv6 = iph;
1700
1701 tcph->check = ~csum_ipv6_magic(&iphv6->saddr, &iphv6->daddr,
1702 len + tcph->doff * 4,
1703 IPPROTO_TCP, 0);
1704 } else {
1705 struct iphdr *iphv4 = iph;
1706
1707 ip_send_check(iphv4);
1708 tcph->check = ~csum_tcpudp_magic(iphv4->saddr, iphv4->daddr,
1709 len + tcph->doff * 4,
1710 IPPROTO_TCP, 0);
1711 }
1712 }
1713
iwl_fill_data_tbs_amsdu(struct iwl_trans * trans,struct sk_buff * skb,struct iwl_txq * txq,u8 hdr_len,struct iwl_cmd_meta * out_meta,struct iwl_device_tx_cmd * dev_cmd,u16 tb1_len)1714 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
1715 struct iwl_txq *txq, u8 hdr_len,
1716 struct iwl_cmd_meta *out_meta,
1717 struct iwl_device_tx_cmd *dev_cmd,
1718 u16 tb1_len)
1719 {
1720 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload;
1721 struct iwl_trans_pcie *trans_pcie =
1722 IWL_TRANS_GET_PCIE_TRANS(txq->trans);
1723 struct ieee80211_hdr *hdr = (void *)skb->data;
1724 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
1725 unsigned int mss = skb_shinfo(skb)->gso_size;
1726 u16 length, iv_len, amsdu_pad;
1727 u8 *start_hdr;
1728 struct iwl_tso_hdr_page *hdr_page;
1729 struct tso_t tso;
1730
1731 /* if the packet is protected, then it must be CCMP or GCMP */
1732 BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN);
1733 iv_len = ieee80211_has_protected(hdr->frame_control) ?
1734 IEEE80211_CCMP_HDR_LEN : 0;
1735
1736 trace_iwlwifi_dev_tx(trans->dev, skb,
1737 iwl_txq_get_tfd(trans, txq, txq->write_ptr),
1738 trans->txqs.tfd.size,
1739 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 0);
1740
1741 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb);
1742 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb);
1743 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len;
1744 amsdu_pad = 0;
1745
1746 /* total amount of header we may need for this A-MSDU */
1747 hdr_room = DIV_ROUND_UP(total_len, mss) *
1748 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len;
1749
1750 /* Our device supports 9 segments at most, it will fit in 1 page */
1751 hdr_page = get_page_hdr(trans, hdr_room, skb);
1752 if (!hdr_page)
1753 return -ENOMEM;
1754
1755 start_hdr = hdr_page->pos;
1756 memcpy(hdr_page->pos, skb->data + hdr_len, iv_len);
1757 hdr_page->pos += iv_len;
1758
1759 /*
1760 * Pull the ieee80211 header + IV to be able to use TSO core,
1761 * we will restore it for the tx_status flow.
1762 */
1763 skb_pull(skb, hdr_len + iv_len);
1764
1765 /*
1766 * Remove the length of all the headers that we don't actually
1767 * have in the MPDU by themselves, but that we duplicate into
1768 * all the different MSDUs inside the A-MSDU.
1769 */
1770 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen);
1771
1772 tso_start(skb, &tso);
1773
1774 while (total_len) {
1775 /* this is the data left for this subframe */
1776 unsigned int data_left =
1777 min_t(unsigned int, mss, total_len);
1778 struct sk_buff *csum_skb = NULL;
1779 unsigned int hdr_tb_len;
1780 dma_addr_t hdr_tb_phys;
1781 struct tcphdr *tcph;
1782 u8 *iph, *subf_hdrs_start = hdr_page->pos;
1783
1784 total_len -= data_left;
1785
1786 memset(hdr_page->pos, 0, amsdu_pad);
1787 hdr_page->pos += amsdu_pad;
1788 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen +
1789 data_left)) & 0x3;
1790 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr));
1791 hdr_page->pos += ETH_ALEN;
1792 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr));
1793 hdr_page->pos += ETH_ALEN;
1794
1795 length = snap_ip_tcp_hdrlen + data_left;
1796 *((__be16 *)hdr_page->pos) = cpu_to_be16(length);
1797 hdr_page->pos += sizeof(length);
1798
1799 /*
1800 * This will copy the SNAP as well which will be considered
1801 * as MAC header.
1802 */
1803 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len);
1804 iph = hdr_page->pos + 8;
1805 tcph = (void *)(iph + ip_hdrlen);
1806
1807 /* For testing on current hardware only */
1808 if (trans_pcie->sw_csum_tx) {
1809 csum_skb = alloc_skb(data_left + tcp_hdrlen(skb),
1810 GFP_ATOMIC);
1811 if (!csum_skb)
1812 return -ENOMEM;
1813
1814 iwl_compute_pseudo_hdr_csum(iph, tcph,
1815 skb->protocol ==
1816 htons(ETH_P_IPV6),
1817 data_left);
1818
1819 skb_put_data(csum_skb, tcph, tcp_hdrlen(skb));
1820 skb_reset_transport_header(csum_skb);
1821 csum_skb->csum_start =
1822 (unsigned char *)tcp_hdr(csum_skb) -
1823 csum_skb->head;
1824 }
1825
1826 hdr_page->pos += snap_ip_tcp_hdrlen;
1827
1828 hdr_tb_len = hdr_page->pos - start_hdr;
1829 hdr_tb_phys = dma_map_single(trans->dev, start_hdr,
1830 hdr_tb_len, DMA_TO_DEVICE);
1831 if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) {
1832 dev_kfree_skb(csum_skb);
1833 return -EINVAL;
1834 }
1835 iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys,
1836 hdr_tb_len, false);
1837 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr,
1838 hdr_tb_phys, hdr_tb_len);
1839 /* add this subframe's headers' length to the tx_cmd */
1840 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start);
1841
1842 /* prepare the start_hdr for the next subframe */
1843 start_hdr = hdr_page->pos;
1844
1845 /* put the payload */
1846 while (data_left) {
1847 unsigned int size = min_t(unsigned int, tso.size,
1848 data_left);
1849 dma_addr_t tb_phys;
1850
1851 if (trans_pcie->sw_csum_tx)
1852 skb_put_data(csum_skb, tso.data, size);
1853
1854 tb_phys = dma_map_single(trans->dev, tso.data,
1855 size, DMA_TO_DEVICE);
1856 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
1857 dev_kfree_skb(csum_skb);
1858 return -EINVAL;
1859 }
1860
1861 iwl_pcie_txq_build_tfd(trans, txq, tb_phys,
1862 size, false);
1863 trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data,
1864 tb_phys, size);
1865
1866 data_left -= size;
1867 tso_build_data(skb, &tso, size);
1868 }
1869
1870 /* For testing on early hardware only */
1871 if (trans_pcie->sw_csum_tx) {
1872 __wsum csum;
1873
1874 csum = skb_checksum(csum_skb,
1875 skb_checksum_start_offset(csum_skb),
1876 csum_skb->len -
1877 skb_checksum_start_offset(csum_skb),
1878 0);
1879 dev_kfree_skb(csum_skb);
1880 dma_sync_single_for_cpu(trans->dev, hdr_tb_phys,
1881 hdr_tb_len, DMA_TO_DEVICE);
1882 tcph->check = csum_fold(csum);
1883 dma_sync_single_for_device(trans->dev, hdr_tb_phys,
1884 hdr_tb_len, DMA_TO_DEVICE);
1885 }
1886 }
1887
1888 /* re -add the WiFi header and IV */
1889 skb_push(skb, hdr_len + iv_len);
1890
1891 return 0;
1892 }
1893 #else /* CONFIG_INET */
iwl_fill_data_tbs_amsdu(struct iwl_trans * trans,struct sk_buff * skb,struct iwl_txq * txq,u8 hdr_len,struct iwl_cmd_meta * out_meta,struct iwl_device_tx_cmd * dev_cmd,u16 tb1_len)1894 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
1895 struct iwl_txq *txq, u8 hdr_len,
1896 struct iwl_cmd_meta *out_meta,
1897 struct iwl_device_tx_cmd *dev_cmd,
1898 u16 tb1_len)
1899 {
1900 /* No A-MSDU without CONFIG_INET */
1901 WARN_ON(1);
1902
1903 return -1;
1904 }
1905 #endif /* CONFIG_INET */
1906
iwl_trans_pcie_tx(struct iwl_trans * trans,struct sk_buff * skb,struct iwl_device_tx_cmd * dev_cmd,int txq_id)1907 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
1908 struct iwl_device_tx_cmd *dev_cmd, int txq_id)
1909 {
1910 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1911 struct ieee80211_hdr *hdr;
1912 struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload;
1913 struct iwl_cmd_meta *out_meta;
1914 struct iwl_txq *txq;
1915 dma_addr_t tb0_phys, tb1_phys, scratch_phys;
1916 void *tb1_addr;
1917 void *tfd;
1918 u16 len, tb1_len;
1919 bool wait_write_ptr;
1920 __le16 fc;
1921 u8 hdr_len;
1922 u16 wifi_seq;
1923 bool amsdu;
1924
1925 txq = trans->txqs.txq[txq_id];
1926
1927 if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used),
1928 "TX on unused queue %d\n", txq_id))
1929 return -EINVAL;
1930
1931 if (unlikely(trans_pcie->sw_csum_tx &&
1932 skb->ip_summed == CHECKSUM_PARTIAL)) {
1933 int offs = skb_checksum_start_offset(skb);
1934 int csum_offs = offs + skb->csum_offset;
1935 __wsum csum;
1936
1937 if (skb_ensure_writable(skb, csum_offs + sizeof(__sum16)))
1938 return -1;
1939
1940 csum = skb_checksum(skb, offs, skb->len - offs, 0);
1941 *(__sum16 *)(skb->data + csum_offs) = csum_fold(csum);
1942
1943 skb->ip_summed = CHECKSUM_UNNECESSARY;
1944 }
1945
1946 if (skb_is_nonlinear(skb) &&
1947 skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) &&
1948 __skb_linearize(skb))
1949 return -ENOMEM;
1950
1951 /* mac80211 always puts the full header into the SKB's head,
1952 * so there's no need to check if it's readable there
1953 */
1954 hdr = (struct ieee80211_hdr *)skb->data;
1955 fc = hdr->frame_control;
1956 hdr_len = ieee80211_hdrlen(fc);
1957
1958 spin_lock(&txq->lock);
1959
1960 if (iwl_txq_space(trans, txq) < txq->high_mark) {
1961 iwl_txq_stop(trans, txq);
1962
1963 /* don't put the packet on the ring, if there is no room */
1964 if (unlikely(iwl_txq_space(trans, txq) < 3)) {
1965 struct iwl_device_tx_cmd **dev_cmd_ptr;
1966
1967 dev_cmd_ptr = (void *)((u8 *)skb->cb +
1968 trans->txqs.dev_cmd_offs);
1969
1970 *dev_cmd_ptr = dev_cmd;
1971 __skb_queue_tail(&txq->overflow_q, skb);
1972
1973 spin_unlock(&txq->lock);
1974 return 0;
1975 }
1976 }
1977
1978 /* In AGG mode, the index in the ring must correspond to the WiFi
1979 * sequence number. This is a HW requirements to help the SCD to parse
1980 * the BA.
1981 * Check here that the packets are in the right place on the ring.
1982 */
1983 wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
1984 WARN_ONCE(txq->ampdu &&
1985 (wifi_seq & 0xff) != txq->write_ptr,
1986 "Q: %d WiFi Seq %d tfdNum %d",
1987 txq_id, wifi_seq, txq->write_ptr);
1988
1989 /* Set up driver data for this TFD */
1990 txq->entries[txq->write_ptr].skb = skb;
1991 txq->entries[txq->write_ptr].cmd = dev_cmd;
1992
1993 dev_cmd->hdr.sequence =
1994 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
1995 INDEX_TO_SEQ(txq->write_ptr)));
1996
1997 tb0_phys = iwl_txq_get_first_tb_dma(txq, txq->write_ptr);
1998 scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) +
1999 offsetof(struct iwl_tx_cmd, scratch);
2000
2001 tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
2002 tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);
2003
2004 /* Set up first empty entry in queue's array of Tx/cmd buffers */
2005 out_meta = &txq->entries[txq->write_ptr].meta;
2006 out_meta->flags = 0;
2007
2008 /*
2009 * The second TB (tb1) points to the remainder of the TX command
2010 * and the 802.11 header - dword aligned size
2011 * (This calculation modifies the TX command, so do it before the
2012 * setup of the first TB)
2013 */
2014 len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) +
2015 hdr_len - IWL_FIRST_TB_SIZE;
2016 /* do not align A-MSDU to dword as the subframe header aligns it */
2017 amsdu = ieee80211_is_data_qos(fc) &&
2018 (*ieee80211_get_qos_ctl(hdr) &
2019 IEEE80211_QOS_CTL_A_MSDU_PRESENT);
2020 if (trans_pcie->sw_csum_tx || !amsdu) {
2021 tb1_len = ALIGN(len, 4);
2022 /* Tell NIC about any 2-byte padding after MAC header */
2023 if (tb1_len != len)
2024 tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD);
2025 } else {
2026 tb1_len = len;
2027 }
2028
2029 /*
2030 * The first TB points to bi-directional DMA data, we'll
2031 * memcpy the data into it later.
2032 */
2033 iwl_pcie_txq_build_tfd(trans, txq, tb0_phys,
2034 IWL_FIRST_TB_SIZE, true);
2035
2036 /* there must be data left over for TB1 or this code must be changed */
2037 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE);
2038
2039 /* map the data for TB1 */
2040 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE;
2041 tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE);
2042 if (unlikely(dma_mapping_error(trans->dev, tb1_phys)))
2043 goto out_err;
2044 iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false);
2045
2046 trace_iwlwifi_dev_tx(trans->dev, skb,
2047 iwl_txq_get_tfd(trans, txq, txq->write_ptr),
2048 trans->txqs.tfd.size,
2049 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len,
2050 hdr_len);
2051
2052 /*
2053 * If gso_size wasn't set, don't give the frame "amsdu treatment"
2054 * (adding subframes, etc.).
2055 * This can happen in some testing flows when the amsdu was already
2056 * pre-built, and we just need to send the resulting skb.
2057 */
2058 if (amsdu && skb_shinfo(skb)->gso_size) {
2059 if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len,
2060 out_meta, dev_cmd,
2061 tb1_len)))
2062 goto out_err;
2063 } else {
2064 struct sk_buff *frag;
2065
2066 if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len,
2067 out_meta)))
2068 goto out_err;
2069
2070 skb_walk_frags(skb, frag) {
2071 if (unlikely(iwl_fill_data_tbs(trans, frag, txq, 0,
2072 out_meta)))
2073 goto out_err;
2074 }
2075 }
2076
2077 /* building the A-MSDU might have changed this data, so memcpy it now */
2078 memcpy(&txq->first_tb_bufs[txq->write_ptr], dev_cmd, IWL_FIRST_TB_SIZE);
2079
2080 tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr);
2081 /* Set up entry for this TFD in Tx byte-count array */
2082 iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len),
2083 iwl_txq_gen1_tfd_get_num_tbs(trans,
2084 tfd));
2085
2086 wait_write_ptr = ieee80211_has_morefrags(fc);
2087
2088 /* start timer if queue currently empty */
2089 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) {
2090 /*
2091 * If the TXQ is active, then set the timer, if not,
2092 * set the timer in remainder so that the timer will
2093 * be armed with the right value when the station will
2094 * wake up.
2095 */
2096 if (!txq->frozen)
2097 mod_timer(&txq->stuck_timer,
2098 jiffies + txq->wd_timeout);
2099 else
2100 txq->frozen_expiry_remainder = txq->wd_timeout;
2101 }
2102
2103 /* Tell device the write index *just past* this latest filled TFD */
2104 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr);
2105 if (!wait_write_ptr)
2106 iwl_pcie_txq_inc_wr_ptr(trans, txq);
2107
2108 /*
2109 * At this point the frame is "transmitted" successfully
2110 * and we will get a TX status notification eventually.
2111 */
2112 spin_unlock(&txq->lock);
2113 return 0;
2114 out_err:
2115 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, txq->write_ptr);
2116 spin_unlock(&txq->lock);
2117 return -1;
2118 }
2119
