1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020 Realtek Corporation
3 */
4
5 #include <linux/pci.h>
6
7 #include "mac.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "ser.h"
11
12 static bool rtw89_pci_disable_clkreq;
13 static bool rtw89_pci_disable_aspm_l1;
14 static bool rtw89_pci_disable_l1ss;
15 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21
rtw89_pci_rst_bdram_pcie(struct rtw89_dev * rtwdev)22 static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev)
23 {
24 u32 val;
25 int ret;
26
27 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1,
28 rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | B_AX_RST_BDRAM);
29
30 ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
31 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
32 rtwdev, R_AX_PCIE_INIT_CFG1);
33
34 if (ret)
35 return -EBUSY;
36
37 return 0;
38 }
39
rtw89_pci_dma_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring,u32 cur_idx,bool tx)40 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
41 struct rtw89_pci_dma_ring *bd_ring,
42 u32 cur_idx, bool tx)
43 {
44 u32 cnt, cur_rp, wp, rp, len;
45
46 rp = bd_ring->rp;
47 wp = bd_ring->wp;
48 len = bd_ring->len;
49
50 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
51 if (tx)
52 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
53 else
54 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
55
56 bd_ring->rp = cur_rp;
57
58 return cnt;
59 }
60
rtw89_pci_txbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)61 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
62 struct rtw89_pci_tx_ring *tx_ring)
63 {
64 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
65 u32 addr_idx = bd_ring->addr.idx;
66 u32 cnt, idx;
67
68 idx = rtw89_read32(rtwdev, addr_idx);
69 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
70
71 return cnt;
72 }
73
rtw89_pci_release_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,u32 cnt,bool release_all)74 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
75 struct rtw89_pci *rtwpci,
76 u32 cnt, bool release_all)
77 {
78 struct rtw89_pci_tx_data *tx_data;
79 struct sk_buff *skb;
80 u32 qlen;
81
82 while (cnt--) {
83 skb = skb_dequeue(&rtwpci->h2c_queue);
84 if (!skb) {
85 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
86 return;
87 }
88 skb_queue_tail(&rtwpci->h2c_release_queue, skb);
89 }
90
91 qlen = skb_queue_len(&rtwpci->h2c_release_queue);
92 if (!release_all)
93 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
94
95 while (qlen--) {
96 skb = skb_dequeue(&rtwpci->h2c_release_queue);
97 if (!skb) {
98 rtw89_err(rtwdev, "failed to release fwcmd\n");
99 return;
100 }
101 tx_data = RTW89_PCI_TX_SKB_CB(skb);
102 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
103 DMA_TO_DEVICE);
104 dev_kfree_skb_any(skb);
105 }
106 }
107
rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)108 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
109 struct rtw89_pci *rtwpci)
110 {
111 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
112 u32 cnt;
113
114 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
115 if (!cnt)
116 return;
117 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
118 }
119
rtw89_pci_rxbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)120 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
121 struct rtw89_pci_rx_ring *rx_ring)
122 {
123 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
124 u32 addr_idx = bd_ring->addr.idx;
125 u32 cnt, idx;
126
127 idx = rtw89_read32(rtwdev, addr_idx);
128 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
129
130 return cnt;
131 }
132
rtw89_pci_sync_skb_for_cpu(struct rtw89_dev * rtwdev,struct sk_buff * skb)133 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
134 struct sk_buff *skb)
135 {
136 struct rtw89_pci_rx_info *rx_info;
137 dma_addr_t dma;
138
139 rx_info = RTW89_PCI_RX_SKB_CB(skb);
140 dma = rx_info->dma;
141 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
142 DMA_FROM_DEVICE);
143 }
144
rtw89_pci_sync_skb_for_device(struct rtw89_dev * rtwdev,struct sk_buff * skb)145 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
146 struct sk_buff *skb)
147 {
148 struct rtw89_pci_rx_info *rx_info;
149 dma_addr_t dma;
150
151 rx_info = RTW89_PCI_RX_SKB_CB(skb);
152 dma = rx_info->dma;
153 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
154 DMA_FROM_DEVICE);
155 }
156
rtw89_pci_rxbd_info_update(struct rtw89_dev * rtwdev,struct sk_buff * skb)157 static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
158 struct sk_buff *skb)
159 {
160 struct rtw89_pci_rxbd_info *rxbd_info;
161 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
162
163 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
164 rx_info->fs = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_FS);
165 rx_info->ls = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_LS);
166 rx_info->len = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_WRITE_SIZE);
167 rx_info->tag = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_TAG);
168
169 return 0;
170 }
171
rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev * rtwdev,bool enable)172 static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
173 {
174 const struct rtw89_pci_info *info = rtwdev->pci_info;
175 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
176 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
177
178 if (enable) {
179 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
180 if (dma_stop2->addr)
181 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
182 } else {
183 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
184 if (dma_stop2->addr)
185 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
186 }
187 }
188
189 static bool
rtw89_skb_put_rx_data(struct rtw89_dev * rtwdev,bool fs,bool ls,struct sk_buff * new,const struct sk_buff * skb,u32 offset,const struct rtw89_pci_rx_info * rx_info,const struct rtw89_rx_desc_info * desc_info)190 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
191 struct sk_buff *new,
192 const struct sk_buff *skb, u32 offset,
193 const struct rtw89_pci_rx_info *rx_info,
194 const struct rtw89_rx_desc_info *desc_info)
195 {
196 u32 copy_len = rx_info->len - offset;
197
198 if (unlikely(skb_tailroom(new) < copy_len)) {
199 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
200 "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
201 rx_info->len, desc_info->pkt_size, offset, fs, ls);
202 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
203 skb->data, rx_info->len);
204 /* length of a single segment skb is desc_info->pkt_size */
205 if (fs && ls) {
206 copy_len = desc_info->pkt_size;
207 } else {
208 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
209 return false;
210 }
211 }
212
213 skb_put_data(new, skb->data + offset, copy_len);
214
215 return true;
216 }
217
rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)218 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
219 struct rtw89_pci_rx_ring *rx_ring)
220 {
221 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
222 struct rtw89_pci_rx_info *rx_info;
223 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
224 struct sk_buff *new = rx_ring->diliver_skb;
225 struct sk_buff *skb;
226 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
227 u32 offset;
228 u32 cnt = 1;
229 bool fs, ls;
230 int ret;
231
232 skb = rx_ring->buf[bd_ring->wp];
233 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
234
235 ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
236 if (ret) {
237 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
238 bd_ring->wp, ret);
239 goto err_sync_device;
240 }
241
242 rx_info = RTW89_PCI_RX_SKB_CB(skb);
243 fs = rx_info->fs;
244 ls = rx_info->ls;
245
246 if (fs) {
247 if (new) {
248 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
249 "skb should not be ready before first segment start\n");
250 goto err_sync_device;
251 }
252 if (desc_info->ready) {
253 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
254 goto err_sync_device;
255 }
256
257 rtw89_core_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
258
259 new = dev_alloc_skb(desc_info->pkt_size);
260 if (!new)
261 goto err_sync_device;
262
263 rx_ring->diliver_skb = new;
264
265 /* first segment has RX desc */
266 offset = desc_info->offset;
267 offset += desc_info->long_rxdesc ? sizeof(struct rtw89_rxdesc_long) :
268 sizeof(struct rtw89_rxdesc_short);
269 } else {
270 offset = sizeof(struct rtw89_pci_rxbd_info);
271 if (!new) {
272 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
273 goto err_sync_device;
274 }
275 }
276 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
277 goto err_sync_device;
278 rtw89_pci_sync_skb_for_device(rtwdev, skb);
279 rtw89_pci_rxbd_increase(rx_ring, 1);
280
281 if (!desc_info->ready) {
282 rtw89_warn(rtwdev, "no rx desc information\n");
283 goto err_free_resource;
284 }
285 if (ls) {
286 rtw89_core_rx(rtwdev, desc_info, new);
287 rx_ring->diliver_skb = NULL;
288 desc_info->ready = false;
289 }
290
291 return cnt;
292
293 err_sync_device:
294 rtw89_pci_sync_skb_for_device(rtwdev, skb);
295 rtw89_pci_rxbd_increase(rx_ring, 1);
296 err_free_resource:
297 if (new)
298 dev_kfree_skb_any(new);
299 rx_ring->diliver_skb = NULL;
300 desc_info->ready = false;
301
302 return cnt;
303 }
304
rtw89_pci_rxbd_deliver(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)305 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
306 struct rtw89_pci_rx_ring *rx_ring,
307 u32 cnt)
308 {
309 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
310 u32 rx_cnt;
311
312 while (cnt && rtwdev->napi_budget_countdown > 0) {
313 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
314 if (!rx_cnt) {
315 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
316
317 /* skip the rest RXBD bufs */
318 rtw89_pci_rxbd_increase(rx_ring, cnt);
319 break;
320 }
321
322 cnt -= rx_cnt;
323 }
324
325 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
326 }
327
rtw89_pci_poll_rxq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)328 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
329 struct rtw89_pci *rtwpci, int budget)
330 {
331 struct rtw89_pci_rx_ring *rx_ring;
332 int countdown = rtwdev->napi_budget_countdown;
333 u32 cnt;
334
335 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
336
337 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
338 if (!cnt)
339 return 0;
340
341 cnt = min_t(u32, budget, cnt);
342
343 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
344
345 /* In case of flushing pending SKBs, the countdown may exceed. */
346 if (rtwdev->napi_budget_countdown <= 0)
347 return budget;
348
349 return budget - countdown;
350 }
351
rtw89_pci_tx_status(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct sk_buff * skb,u8 tx_status)352 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
353 struct rtw89_pci_tx_ring *tx_ring,
354 struct sk_buff *skb, u8 tx_status)
355 {
356 struct ieee80211_tx_info *info;
357
358 info = IEEE80211_SKB_CB(skb);
359 ieee80211_tx_info_clear_status(info);
360
361 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
362 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
363 if (tx_status == RTW89_TX_DONE) {
364 info->flags |= IEEE80211_TX_STAT_ACK;
365 tx_ring->tx_acked++;
366 } else {
367 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
368 rtw89_debug(rtwdev, RTW89_DBG_FW,
369 "failed to TX of status %x\n", tx_status);
370 switch (tx_status) {
371 case RTW89_TX_RETRY_LIMIT:
372 tx_ring->tx_retry_lmt++;
373 break;
374 case RTW89_TX_LIFE_TIME:
375 tx_ring->tx_life_time++;
376 break;
377 case RTW89_TX_MACID_DROP:
378 tx_ring->tx_mac_id_drop++;
379 break;
380 default:
381 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
382 break;
383 }
384 }
385
386 ieee80211_tx_status_ni(rtwdev->hw, skb);
387 }
388
rtw89_pci_reclaim_txbd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)389 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
390 {
391 struct rtw89_pci_tx_wd *txwd;
392 u32 cnt;
393
394 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
395 while (cnt--) {
396 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
397 if (!txwd) {
398 rtw89_warn(rtwdev, "No busy txwd pages available\n");
399 break;
400 }
401
402 list_del_init(&txwd->list);
403
404 /* this skb has been freed by RPP */
405 if (skb_queue_len(&txwd->queue) == 0)
406 rtw89_pci_enqueue_txwd(tx_ring, txwd);
407 }
408 }
409
rtw89_pci_release_busy_txwd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)410 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
411 struct rtw89_pci_tx_ring *tx_ring)
412 {
413 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
414 struct rtw89_pci_tx_wd *txwd;
415 int i;
416
417 for (i = 0; i < wd_ring->page_num; i++) {
418 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
419 if (!txwd)
420 break;
421
422 list_del_init(&txwd->list);
423 }
424 }
425
rtw89_pci_release_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,u16 seq,u8 tx_status)426 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
427 struct rtw89_pci_tx_ring *tx_ring,
428 struct rtw89_pci_tx_wd *txwd, u16 seq,
429 u8 tx_status)
430 {
431 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
432 struct rtw89_pci_tx_data *tx_data;
433 struct sk_buff *skb, *tmp;
434 u8 txch = tx_ring->txch;
435
436 if (!list_empty(&txwd->list)) {
437 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
438 /* In low power mode, RPP can receive before updating of TX BD.
439 * In normal mode, it should not happen so give it a warning.
440 */
441 if (!rtwpci->low_power && !list_empty(&txwd->list))
442 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
443 txch, seq);
444 }
445
446 skb_queue_walk_safe(&txwd->queue, skb, tmp) {
447 skb_unlink(skb, &txwd->queue);
448
449 tx_data = RTW89_PCI_TX_SKB_CB(skb);
450 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
451 DMA_TO_DEVICE);
452
453 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
454 }
455
456 if (list_empty(&txwd->list))
457 rtw89_pci_enqueue_txwd(tx_ring, txwd);
458 }
459
rtw89_pci_release_rpp(struct rtw89_dev * rtwdev,struct rtw89_pci_rpp_fmt * rpp)460 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
461 struct rtw89_pci_rpp_fmt *rpp)
462 {
463 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
464 struct rtw89_pci_tx_ring *tx_ring;
465 struct rtw89_pci_tx_wd_ring *wd_ring;
466 struct rtw89_pci_tx_wd *txwd;
467 u16 seq;
468 u8 qsel, tx_status, txch;
469
470 seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
471 qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
472 tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
473 txch = rtw89_core_get_ch_dma(rtwdev, qsel);
474
475 if (txch == RTW89_TXCH_CH12) {
476 rtw89_warn(rtwdev, "should no fwcmd release report\n");
477 return;
478 }
479
480 tx_ring = &rtwpci->tx_rings[txch];
481 wd_ring = &tx_ring->wd_ring;
482 txwd = &wd_ring->pages[seq];
483
484 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
485 }
486
rtw89_pci_release_pending_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)487 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
488 struct rtw89_pci_tx_ring *tx_ring)
489 {
490 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
491 struct rtw89_pci_tx_wd *txwd;
492 int i;
493
494 for (i = 0; i < wd_ring->page_num; i++) {
495 txwd = &wd_ring->pages[i];
496
497 if (!list_empty(&txwd->list))
498 continue;
499
500 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
501 }
502 }
503
rtw89_pci_release_tx_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 max_cnt)504 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
505 struct rtw89_pci_rx_ring *rx_ring,
506 u32 max_cnt)
507 {
508 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
509 struct rtw89_pci_rx_info *rx_info;
510 struct rtw89_pci_rpp_fmt *rpp;
511 struct rtw89_rx_desc_info desc_info = {};
512 struct sk_buff *skb;
513 u32 cnt = 0;
514 u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
515 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
516 u32 offset;
517 int ret;
518
519 skb = rx_ring->buf[bd_ring->wp];
520 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
521
522 ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
523 if (ret) {
524 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
525 bd_ring->wp, ret);
526 goto err_sync_device;
527 }
528
529 rx_info = RTW89_PCI_RX_SKB_CB(skb);
530 if (!rx_info->fs || !rx_info->ls) {
531 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
532 return cnt;
533 }
534
535 rtw89_core_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
536
537 /* first segment has RX desc */
538 offset = desc_info.offset;
539 offset += desc_info.long_rxdesc ? sizeof(struct rtw89_rxdesc_long) :
540 sizeof(struct rtw89_rxdesc_short);
541 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
542 rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
543 rtw89_pci_release_rpp(rtwdev, rpp);
544 }
545
546 rtw89_pci_sync_skb_for_device(rtwdev, skb);
547 rtw89_pci_rxbd_increase(rx_ring, 1);
548 cnt++;
549
550 return cnt;
551
552 err_sync_device:
553 rtw89_pci_sync_skb_for_device(rtwdev, skb);
554 return 0;
555 }
556
rtw89_pci_release_tx(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)557 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
558 struct rtw89_pci_rx_ring *rx_ring,
559 u32 cnt)
560 {
561 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
562 u32 release_cnt;
563
564 while (cnt) {
565 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
566 if (!release_cnt) {
567 rtw89_err(rtwdev, "failed to release TX skbs\n");
568
569 /* skip the rest RXBD bufs */
570 rtw89_pci_rxbd_increase(rx_ring, cnt);
571 break;
572 }
573
574 cnt -= release_cnt;
575 }
576
577 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
578 }
579
rtw89_pci_poll_rpq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)580 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
581 struct rtw89_pci *rtwpci, int budget)
582 {
583 struct rtw89_pci_rx_ring *rx_ring;
584 u32 cnt;
585 int work_done;
586
587 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
588
589 spin_lock_bh(&rtwpci->trx_lock);
590
591 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
592 if (cnt == 0)
593 goto out_unlock;
594
595 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
596
597 out_unlock:
598 spin_unlock_bh(&rtwpci->trx_lock);
599
600 /* always release all RPQ */
601 work_done = min_t(int, cnt, budget);
602 rtwdev->napi_budget_countdown -= work_done;
603
604 return work_done;
605 }
606
rtw89_pci_isr_rxd_unavail(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)607 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
608 struct rtw89_pci *rtwpci)
609 {
610 struct rtw89_pci_rx_ring *rx_ring;
611 struct rtw89_pci_dma_ring *bd_ring;
612 u32 reg_idx;
613 u16 hw_idx, hw_idx_next, host_idx;
614 int i;
615
616 for (i = 0; i < RTW89_RXCH_NUM; i++) {
617 rx_ring = &rtwpci->rx_rings[i];
618 bd_ring = &rx_ring->bd_ring;
619
620 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
621 hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
622 host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
623 hw_idx_next = (hw_idx + 1) % bd_ring->len;
624
625 if (hw_idx_next == host_idx)
626 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
627
628 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
629 "%d RXD unavailable, idx=0x%08x, len=%d\n",
630 i, reg_idx, bd_ring->len);
631 }
632 }
633
rtw89_pci_recognize_intrs(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)634 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
635 struct rtw89_pci *rtwpci,
636 struct rtw89_pci_isrs *isrs)
637 {
638 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
639 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
640 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
641
642 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
643 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
644 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
645 }
646 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
647
rtw89_pci_recognize_intrs_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)648 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
649 struct rtw89_pci *rtwpci,
650 struct rtw89_pci_isrs *isrs)
651 {
652 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
653 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
654 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
655 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
656 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
657 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
658 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
659
660 if (isrs->halt_c2h_isrs)
661 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
662 if (isrs->isrs[0])
663 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
664 if (isrs->isrs[1])
665 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
666 }
667 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
668
rtw89_pci_clear_isr0(struct rtw89_dev * rtwdev,u32 isr00)669 static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00)
670 {
671 /* write 1 clear */
672 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isr00);
673 }
674
rtw89_pci_enable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)675 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
676 {
677 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
678 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
679 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
680 }
681 EXPORT_SYMBOL(rtw89_pci_enable_intr);
682
rtw89_pci_disable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)683 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
684 {
685 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
686 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
687 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
688 }
689 EXPORT_SYMBOL(rtw89_pci_disable_intr);
690
rtw89_pci_enable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)691 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
692 {
693 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
694 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
695 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
696 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
697 }
698 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
699
rtw89_pci_disable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)700 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
701 {
702 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
703 }
704 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
705
rtw89_pci_ops_recovery_start(struct rtw89_dev * rtwdev)706 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
707 {
708 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
709 unsigned long flags;
710
711 spin_lock_irqsave(&rtwpci->irq_lock, flags);
712 rtw89_chip_disable_intr(rtwdev, rtwpci);
713 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
714 rtw89_chip_enable_intr(rtwdev, rtwpci);
715 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
716 }
717
rtw89_pci_ops_recovery_complete(struct rtw89_dev * rtwdev)718 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
719 {
720 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
721 unsigned long flags;
722
723 spin_lock_irqsave(&rtwpci->irq_lock, flags);
724 rtw89_chip_disable_intr(rtwdev, rtwpci);
725 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
726 rtw89_chip_enable_intr(rtwdev, rtwpci);
727 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
728 }
729
rtw89_pci_low_power_interrupt_handler(struct rtw89_dev * rtwdev)730 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
731 {
732 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
733 int budget = NAPI_POLL_WEIGHT;
734
735 /* To prevent RXQ get stuck due to run out of budget. */
736 rtwdev->napi_budget_countdown = budget;
737
738 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
739 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
740 }
741
rtw89_pci_interrupt_threadfn(int irq,void * dev)742 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
743 {
744 struct rtw89_dev *rtwdev = dev;
745 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
746 struct rtw89_pci_isrs isrs;
747 unsigned long flags;
748
749 spin_lock_irqsave(&rtwpci->irq_lock, flags);
750 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
751 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
752
753 if (unlikely(isrs.isrs[0] & B_AX_RDU_INT))
754 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
755
756 if (unlikely(isrs.halt_c2h_isrs & B_AX_HALT_C2H_INT_EN))
757 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
758
759 if (unlikely(isrs.halt_c2h_isrs & B_AX_WDT_TIMEOUT_INT_EN))
760 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
761
762 if (unlikely(rtwpci->under_recovery))
763 goto enable_intr;
764
765 if (unlikely(rtwpci->low_power)) {
766 rtw89_pci_low_power_interrupt_handler(rtwdev);
767 goto enable_intr;
768 }
769
770 if (likely(rtwpci->running)) {
771 local_bh_disable();
772 napi_schedule(&rtwdev->napi);
773 local_bh_enable();
774 }
775
776 return IRQ_HANDLED;
777
778 enable_intr:
779 spin_lock_irqsave(&rtwpci->irq_lock, flags);
780 if (likely(rtwpci->running))
781 rtw89_chip_enable_intr(rtwdev, rtwpci);
782 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
783 return IRQ_HANDLED;
784 }
785
rtw89_pci_interrupt_handler(int irq,void * dev)786 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
787 {
788 struct rtw89_dev *rtwdev = dev;
789 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
790 unsigned long flags;
791 irqreturn_t irqret = IRQ_WAKE_THREAD;
792
793 spin_lock_irqsave(&rtwpci->irq_lock, flags);
794
795 /* If interrupt event is on the road, it is still trigger interrupt
796 * even we have done pci_stop() to turn off IMR.
797 */
798 if (unlikely(!rtwpci->running)) {
799 irqret = IRQ_HANDLED;
800 goto exit;
801 }
802
803 rtw89_chip_disable_intr(rtwdev, rtwpci);
804 exit:
805 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
806
807 return irqret;
808 }
809
810 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
811 [RTW89_TXCH_##txch] = { \
812 .num = R_AX_##txch##_TXBD_NUM ##v, \
813 .idx = R_AX_##txch##_TXBD_IDX ##v, \
814 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
815 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
816 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
817 }
818
819 #define DEF_TXCHADDRS(info, txch, v...) \
820 [RTW89_TXCH_##txch] = { \
821 .num = R_AX_##txch##_TXBD_NUM, \
822 .idx = R_AX_##txch##_TXBD_IDX, \
823 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
824 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
825 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
826 }
827
828 #define DEF_RXCHADDRS(info, rxch, v...) \
829 [RTW89_RXCH_##rxch] = { \
830 .num = R_AX_##rxch##_RXBD_NUM ##v, \
831 .idx = R_AX_##rxch##_RXBD_IDX ##v, \
832 .desa_l = R_AX_##rxch##_RXBD_DESA_L ##v, \
833 .desa_h = R_AX_##rxch##_RXBD_DESA_H ##v, \
834 }
835
836 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
837 .tx = {
838 DEF_TXCHADDRS(info, ACH0),
839 DEF_TXCHADDRS(info, ACH1),
840 DEF_TXCHADDRS(info, ACH2),
841 DEF_TXCHADDRS(info, ACH3),
842 DEF_TXCHADDRS(info, ACH4),
843 DEF_TXCHADDRS(info, ACH5),
844 DEF_TXCHADDRS(info, ACH6),
845 DEF_TXCHADDRS(info, ACH7),
846 DEF_TXCHADDRS(info, CH8),
847 DEF_TXCHADDRS(info, CH9),
848 DEF_TXCHADDRS_TYPE1(info, CH10),
849 DEF_TXCHADDRS_TYPE1(info, CH11),
850 DEF_TXCHADDRS(info, CH12),
851 },
852 .rx = {
853 DEF_RXCHADDRS(info, RXQ),
854 DEF_RXCHADDRS(info, RPQ),
855 },
856 };
857 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
858
859 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
860 .tx = {
861 DEF_TXCHADDRS(info, ACH0, _V1),
862 DEF_TXCHADDRS(info, ACH1, _V1),
863 DEF_TXCHADDRS(info, ACH2, _V1),
864 DEF_TXCHADDRS(info, ACH3, _V1),
865 DEF_TXCHADDRS(info, ACH4, _V1),
866 DEF_TXCHADDRS(info, ACH5, _V1),
867 DEF_TXCHADDRS(info, ACH6, _V1),
868 DEF_TXCHADDRS(info, ACH7, _V1),
869 DEF_TXCHADDRS(info, CH8, _V1),
870 DEF_TXCHADDRS(info, CH9, _V1),
871 DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
872 DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
873 DEF_TXCHADDRS(info, CH12, _V1),
874 },
875 .rx = {
876 DEF_RXCHADDRS(info, RXQ, _V1),
877 DEF_RXCHADDRS(info, RPQ, _V1),
878 },
879 };
880 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
881
882 #undef DEF_TXCHADDRS_TYPE1
883 #undef DEF_TXCHADDRS
884 #undef DEF_RXCHADDRS
885
rtw89_pci_get_txch_addrs(struct rtw89_dev * rtwdev,enum rtw89_tx_channel txch,const struct rtw89_pci_ch_dma_addr ** addr)886 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
887 enum rtw89_tx_channel txch,
888 const struct rtw89_pci_ch_dma_addr **addr)
889 {
890 const struct rtw89_pci_info *info = rtwdev->pci_info;
891
892 if (txch >= RTW89_TXCH_NUM)
893 return -EINVAL;
894
895 *addr = &info->dma_addr_set->tx[txch];
896
897 return 0;
898 }
899
rtw89_pci_get_rxch_addrs(struct rtw89_dev * rtwdev,enum rtw89_rx_channel rxch,const struct rtw89_pci_ch_dma_addr ** addr)900 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
901 enum rtw89_rx_channel rxch,
902 const struct rtw89_pci_ch_dma_addr **addr)
903 {
904 const struct rtw89_pci_info *info = rtwdev->pci_info;
905
906 if (rxch >= RTW89_RXCH_NUM)
907 return -EINVAL;
908
909 *addr = &info->dma_addr_set->rx[rxch];
910
911 return 0;
912 }
913
rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring * ring)914 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
915 {
916 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
917
918 /* reserved 1 desc check ring is full or not */
919 if (bd_ring->rp > bd_ring->wp)
920 return bd_ring->rp - bd_ring->wp - 1;
921
922 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
923 }
924
925 static
__rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev * rtwdev)926 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
927 {
928 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
929 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
930 u32 cnt;
931
932 spin_lock_bh(&rtwpci->trx_lock);
933 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
934 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
935 spin_unlock_bh(&rtwpci->trx_lock);
936
937 return cnt;
938 }
939
940 static
__rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev * rtwdev,u8 txch)941 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
942 u8 txch)
943 {
944 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
945 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
946 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
947 u32 cnt;
948
949 spin_lock_bh(&rtwpci->trx_lock);
950 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
951 cnt = min(cnt, wd_ring->curr_num);
952 spin_unlock_bh(&rtwpci->trx_lock);
953
954 return cnt;
955 }
956
__rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)957 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
958 u8 txch)
959 {
960 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
961 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
962 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
963 u32 bd_cnt, wd_cnt, min_cnt = 0;
964 struct rtw89_pci_rx_ring *rx_ring;
965 u32 cnt;
966
967 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
968
969 spin_lock_bh(&rtwpci->trx_lock);
970 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
971 wd_cnt = wd_ring->curr_num;
972
973 if (wd_cnt == 0 || bd_cnt == 0) {
974 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
975 if (cnt)
976 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
977 else if (wd_cnt == 0)
978 goto out_unlock;
979
980 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
981 if (bd_cnt == 0)
982 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
983 }
984
985 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
986 wd_cnt = wd_ring->curr_num;
987 min_cnt = min(bd_cnt, wd_cnt);
988 if (min_cnt == 0)
989 rtw89_debug(rtwdev, rtwpci->low_power ? RTW89_DBG_TXRX : RTW89_DBG_UNEXP,
990 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
991 wd_cnt, bd_cnt);
992
993 out_unlock:
994 spin_unlock_bh(&rtwpci->trx_lock);
995
996 return min_cnt;
997 }
998
rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)999 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1000 u8 txch)
1001 {
1002 if (rtwdev->hci.paused)
1003 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1004
1005 if (txch == RTW89_TXCH_CH12)
1006 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1007
1008 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1009 }
1010
__rtw89_pci_tx_kick_off(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1011 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1012 {
1013 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1014 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1015 u32 host_idx, addr;
1016
1017 spin_lock_bh(&rtwpci->trx_lock);
1018
1019 addr = bd_ring->addr.idx;
1020 host_idx = bd_ring->wp;
1021 rtw89_write16(rtwdev, addr, host_idx);
1022
1023 spin_unlock_bh(&rtwpci->trx_lock);
1024 }
1025
rtw89_pci_tx_bd_ring_update(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,int n_txbd)1026 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1027 int n_txbd)
1028 {
1029 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1030 u32 host_idx, len;
1031
1032 len = bd_ring->len;
1033 host_idx = bd_ring->wp + n_txbd;
1034 host_idx = host_idx < len ? host_idx : host_idx - len;
1035
1036 bd_ring->wp = host_idx;
1037 }
1038
rtw89_pci_ops_tx_kick_off(struct rtw89_dev * rtwdev,u8 txch)1039 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1040 {
1041 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1042 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1043
1044 if (rtwdev->hci.paused) {
1045 set_bit(txch, rtwpci->kick_map);
1046 return;
1047 }
1048
1049 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1050 }
1051
rtw89_pci_tx_kick_off_pending(struct rtw89_dev * rtwdev)1052 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1053 {
1054 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1055 struct rtw89_pci_tx_ring *tx_ring;
1056 int txch;
1057
1058 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1059 if (!test_and_clear_bit(txch, rtwpci->kick_map))
1060 continue;
1061
1062 tx_ring = &rtwpci->tx_rings[txch];
1063 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1064 }
1065 }
1066
__pci_flush_txch(struct rtw89_dev * rtwdev,u8 txch,bool drop)1067 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1068 {
1069 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1070 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1071 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1072 u32 cur_idx, cur_rp;
1073 u8 i;
1074
1075 /* Because the time taked by the I/O is a bit dynamic, it's hard to
1076 * define a reasonable fixed total timeout to use read_poll_timeout*
1077 * helper. Instead, we can ensure a reasonable polling times, so we
1078 * just use for loop with udelay here.
1079 */
1080 for (i = 0; i < 60; i++) {
1081 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1082 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1083 if (cur_rp == bd_ring->wp)
1084 return;
1085
1086 udelay(1);
1087 }
1088
1089 if (!drop)
1090 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1091 }
1092
__rtw89_pci_ops_flush_txchs(struct rtw89_dev * rtwdev,u32 txchs,bool drop)1093 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1094 bool drop)
1095 {
1096 const struct rtw89_pci_info *info = rtwdev->pci_info;
1097 u8 i;
1098
1099 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1100 /* It may be unnecessary to flush FWCMD queue. */
1101 if (i == RTW89_TXCH_CH12)
1102 continue;
1103 if (info->tx_dma_ch_mask & BIT(i))
1104 continue;
1105
1106 if (txchs & BIT(i))
1107 __pci_flush_txch(rtwdev, i, drop);
1108 }
1109 }
1110
rtw89_pci_ops_flush_queues(struct rtw89_dev * rtwdev,u32 queues,bool drop)1111 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1112 bool drop)
1113 {
1114 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1115 }
1116
rtw89_pci_fill_txaddr_info(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1117 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1118 void *txaddr_info_addr, u32 total_len,
1119 dma_addr_t dma, u8 *add_info_nr)
1120 {
1121 struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1122
1123 txaddr_info->length = cpu_to_le16(total_len);
1124 txaddr_info->option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS |
1125 RTW89_PCI_ADDR_NUM(1));
1126 txaddr_info->dma = cpu_to_le32(dma);
1127
1128 *add_info_nr = 1;
1129
1130 return sizeof(*txaddr_info);
1131 }
1132 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1133
rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1134 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1135 void *txaddr_info_addr, u32 total_len,
1136 dma_addr_t dma, u8 *add_info_nr)
1137 {
1138 struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1139 u32 remain = total_len;
1140 u32 len;
1141 u16 length_option;
1142 int n;
1143
1144 for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1145 len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1146 TXADDR_INFO_LENTHG_V1_MAX : remain;
1147 remain -= len;
1148
1149 length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1150 FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1151 FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1152 txaddr_info->length_opt = cpu_to_le16(length_option);
1153 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1154 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1155
1156 dma += len;
1157 txaddr_info++;
1158 }
1159
1160 WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1161 remain, total_len);
1162
1163 *add_info_nr = n;
1164
1165 return n * sizeof(*txaddr_info);
1166 }
1167 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1168
rtw89_pci_txwd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,struct rtw89_core_tx_request * tx_req)1169 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1170 struct rtw89_pci_tx_ring *tx_ring,
1171 struct rtw89_pci_tx_wd *txwd,
1172 struct rtw89_core_tx_request *tx_req)
1173 {
1174 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1175 const struct rtw89_chip_info *chip = rtwdev->chip;
1176 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1177 struct rtw89_txwd_info *txwd_info;
1178 struct rtw89_pci_tx_wp_info *txwp_info;
1179 void *txaddr_info_addr;
1180 struct pci_dev *pdev = rtwpci->pdev;
1181 struct sk_buff *skb = tx_req->skb;
1182 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1183 bool en_wd_info = desc_info->en_wd_info;
1184 u32 txwd_len;
1185 u32 txwp_len;
1186 u32 txaddr_info_len;
1187 dma_addr_t dma;
1188 int ret;
1189
1190 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1191 if (dma_mapping_error(&pdev->dev, dma)) {
1192 rtw89_err(rtwdev, "failed to map skb dma data\n");
1193 ret = -EBUSY;
1194 goto err;
1195 }
1196
1197 tx_data->dma = dma;
1198
1199 txwp_len = sizeof(*txwp_info);
1200 txwd_len = chip->txwd_body_size;
1201 txwd_len += en_wd_info ? sizeof(*txwd_info) : 0;
1202
1203 txwp_info = txwd->vaddr + txwd_len;
1204 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1205 txwp_info->seq1 = 0;
1206 txwp_info->seq2 = 0;
1207 txwp_info->seq3 = 0;
1208
1209 tx_ring->tx_cnt++;
1210 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1211 txaddr_info_len =
1212 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1213 dma, &desc_info->addr_info_nr);
1214
1215 txwd->len = txwd_len + txwp_len + txaddr_info_len;
1216
1217 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1218
1219 skb_queue_tail(&txwd->queue, skb);
1220
1221 return 0;
1222
1223 err:
1224 return ret;
1225 }
1226
rtw89_pci_fwcmd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1227 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1228 struct rtw89_pci_tx_ring *tx_ring,
1229 struct rtw89_pci_tx_bd_32 *txbd,
1230 struct rtw89_core_tx_request *tx_req)
1231 {
1232 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1233 const struct rtw89_chip_info *chip = rtwdev->chip;
1234 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1235 void *txdesc;
1236 int txdesc_size = chip->h2c_desc_size;
1237 struct pci_dev *pdev = rtwpci->pdev;
1238 struct sk_buff *skb = tx_req->skb;
1239 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1240 dma_addr_t dma;
1241
1242 txdesc = skb_push(skb, txdesc_size);
1243 memset(txdesc, 0, txdesc_size);
1244 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1245
1246 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1247 if (dma_mapping_error(&pdev->dev, dma)) {
1248 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1249 return -EBUSY;
1250 }
1251
1252 tx_data->dma = dma;
1253 txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1254 txbd->length = cpu_to_le16(skb->len);
1255 txbd->dma = cpu_to_le32(tx_data->dma);
1256 skb_queue_tail(&rtwpci->h2c_queue, skb);
1257
1258 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1259
1260 return 0;
1261 }
1262
rtw89_pci_txbd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1263 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1264 struct rtw89_pci_tx_ring *tx_ring,
1265 struct rtw89_pci_tx_bd_32 *txbd,
1266 struct rtw89_core_tx_request *tx_req)
1267 {
1268 struct rtw89_pci_tx_wd *txwd;
1269 int ret;
1270
1271 /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1272 * buffer with WD BODY only. So here we don't need to check the free
1273 * pages of the wd ring.
1274 */
1275 if (tx_ring->txch == RTW89_TXCH_CH12)
1276 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1277
1278 txwd = rtw89_pci_dequeue_txwd(tx_ring);
1279 if (!txwd) {
1280 rtw89_err(rtwdev, "no available TXWD\n");
1281 ret = -ENOSPC;
1282 goto err;
1283 }
1284
1285 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1286 if (ret) {
1287 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1288 goto err_enqueue_wd;
1289 }
1290
1291 list_add_tail(&txwd->list, &tx_ring->busy_pages);
1292
1293 txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1294 txbd->length = cpu_to_le16(txwd->len);
1295 txbd->dma = cpu_to_le32(txwd->paddr);
1296
1297 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1298
1299 return 0;
1300
1301 err_enqueue_wd:
1302 rtw89_pci_enqueue_txwd(tx_ring, txwd);
1303 err:
1304 return ret;
1305 }
1306
rtw89_pci_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req,u8 txch)1307 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1308 u8 txch)
1309 {
1310 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1311 struct rtw89_pci_tx_ring *tx_ring;
1312 struct rtw89_pci_tx_bd_32 *txbd;
1313 u32 n_avail_txbd;
1314 int ret = 0;
1315
1316 /* check the tx type and dma channel for fw cmd queue */
1317 if ((txch == RTW89_TXCH_CH12 ||
1318 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1319 (txch != RTW89_TXCH_CH12 ||
1320 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1321 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1322 return -EINVAL;
1323 }
1324
1325 tx_ring = &rtwpci->tx_rings[txch];
1326 spin_lock_bh(&rtwpci->trx_lock);
1327
1328 n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1329 if (n_avail_txbd == 0) {
1330 rtw89_err(rtwdev, "no available TXBD\n");
1331 ret = -ENOSPC;
1332 goto err_unlock;
1333 }
1334
1335 txbd = rtw89_pci_get_next_txbd(tx_ring);
1336 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1337 if (ret) {
1338 rtw89_err(rtwdev, "failed to submit TXBD\n");
1339 goto err_unlock;
1340 }
1341
1342 spin_unlock_bh(&rtwpci->trx_lock);
1343 return 0;
1344
1345 err_unlock:
1346 spin_unlock_bh(&rtwpci->trx_lock);
1347 return ret;
1348 }
1349
rtw89_pci_ops_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req)1350 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1351 {
1352 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1353 int ret;
1354
1355 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1356 if (ret) {
1357 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1358 return ret;
1359 }
1360
1361 return 0;
1362 }
1363
1364 static const struct rtw89_pci_bd_ram bd_ram_table[RTW89_TXCH_NUM] = {
1365 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1366 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1367 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1368 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1369 [RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1370 [RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1371 [RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1372 [RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1373 [RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1},
1374 [RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1},
1375 [RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1376 [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1377 [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1378 };
1379
rtw89_pci_reset_trx_rings(struct rtw89_dev * rtwdev)1380 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1381 {
1382 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1383 const struct rtw89_pci_info *info = rtwdev->pci_info;
1384 struct rtw89_pci_tx_ring *tx_ring;
1385 struct rtw89_pci_rx_ring *rx_ring;
1386 struct rtw89_pci_dma_ring *bd_ring;
1387 const struct rtw89_pci_bd_ram *bd_ram;
1388 u32 addr_num;
1389 u32 addr_bdram;
1390 u32 addr_desa_l;
1391 u32 val32;
1392 int i;
1393
1394 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1395 if (info->tx_dma_ch_mask & BIT(i))
1396 continue;
1397
1398 tx_ring = &rtwpci->tx_rings[i];
1399 bd_ring = &tx_ring->bd_ring;
1400 bd_ram = &bd_ram_table[i];
1401 addr_num = bd_ring->addr.num;
1402 addr_bdram = bd_ring->addr.bdram;
1403 addr_desa_l = bd_ring->addr.desa_l;
1404 bd_ring->wp = 0;
1405 bd_ring->rp = 0;
1406
1407 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1408 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1409 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1410
1411 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1412 rtw89_write32(rtwdev, addr_bdram, val32);
1413 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1414 }
1415
1416 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1417 rx_ring = &rtwpci->rx_rings[i];
1418 bd_ring = &rx_ring->bd_ring;
1419 addr_num = bd_ring->addr.num;
1420 addr_desa_l = bd_ring->addr.desa_l;
1421 bd_ring->wp = 0;
1422 bd_ring->rp = 0;
1423 rx_ring->diliver_skb = NULL;
1424 rx_ring->diliver_desc.ready = false;
1425
1426 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1427 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1428 }
1429 }
1430
rtw89_pci_release_tx_ring(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1431 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1432 struct rtw89_pci_tx_ring *tx_ring)
1433 {
1434 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1435 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1436 }
1437
rtw89_pci_ops_reset(struct rtw89_dev * rtwdev)1438 static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1439 {
1440 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1441 const struct rtw89_pci_info *info = rtwdev->pci_info;
1442 int txch;
1443
1444 rtw89_pci_reset_trx_rings(rtwdev);
1445
1446 spin_lock_bh(&rtwpci->trx_lock);
1447 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1448 if (info->tx_dma_ch_mask & BIT(txch))
1449 continue;
1450 if (txch == RTW89_TXCH_CH12) {
1451 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1452 skb_queue_len(&rtwpci->h2c_queue), true);
1453 continue;
1454 }
1455 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1456 }
1457 spin_unlock_bh(&rtwpci->trx_lock);
1458 }
1459
rtw89_pci_enable_intr_lock(struct rtw89_dev * rtwdev)1460 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1461 {
1462 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1463 unsigned long flags;
1464
1465 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1466 rtwpci->running = true;
1467 rtw89_chip_enable_intr(rtwdev, rtwpci);
1468 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1469 }
1470
rtw89_pci_disable_intr_lock(struct rtw89_dev * rtwdev)1471 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1472 {
1473 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1474 unsigned long flags;
1475
1476 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1477 rtwpci->running = false;
1478 rtw89_chip_disable_intr(rtwdev, rtwpci);
1479 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1480 }
1481
rtw89_pci_ops_start(struct rtw89_dev * rtwdev)1482 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1483 {
1484 rtw89_core_napi_start(rtwdev);
1485 rtw89_pci_enable_intr_lock(rtwdev);
1486
1487 return 0;
1488 }
1489
rtw89_pci_ops_stop(struct rtw89_dev * rtwdev)1490 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1491 {
1492 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1493 struct pci_dev *pdev = rtwpci->pdev;
1494
1495 rtw89_pci_disable_intr_lock(rtwdev);
1496 synchronize_irq(pdev->irq);
1497 rtw89_core_napi_stop(rtwdev);
1498 }
1499
rtw89_pci_ops_pause(struct rtw89_dev * rtwdev,bool pause)1500 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1501 {
1502 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1503 struct pci_dev *pdev = rtwpci->pdev;
1504
1505 if (pause) {
1506 rtw89_pci_disable_intr_lock(rtwdev);
1507 synchronize_irq(pdev->irq);
1508 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1509 napi_synchronize(&rtwdev->napi);
1510 } else {
1511 rtw89_pci_enable_intr_lock(rtwdev);
1512 rtw89_pci_tx_kick_off_pending(rtwdev);
1513 }
1514 }
1515
1516 static
rtw89_pci_switch_bd_idx_addr(struct rtw89_dev * rtwdev,bool low_power)1517 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1518 {
1519 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1520 const struct rtw89_pci_info *info = rtwdev->pci_info;
1521 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1522 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1523 struct rtw89_pci_tx_ring *tx_ring;
1524 struct rtw89_pci_rx_ring *rx_ring;
1525 int i;
1526
1527 if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1528 return;
1529
1530 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1531 tx_ring = &rtwpci->tx_rings[i];
1532 tx_ring->bd_ring.addr.idx = low_power ?
1533 bd_idx_addr->tx_bd_addrs[i] :
1534 dma_addr_set->tx[i].idx;
1535 }
1536
1537 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1538 rx_ring = &rtwpci->rx_rings[i];
1539 rx_ring->bd_ring.addr.idx = low_power ?
1540 bd_idx_addr->rx_bd_addrs[i] :
1541 dma_addr_set->rx[i].idx;
1542 }
1543 }
1544
rtw89_pci_ops_switch_mode(struct rtw89_dev * rtwdev,bool low_power)1545 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1546 {
1547 enum rtw89_pci_intr_mask_cfg cfg;
1548
1549 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1550
1551 cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1552 rtw89_chip_config_intr_mask(rtwdev, cfg);
1553 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1554 }
1555
1556 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1557
rtw89_pci_ops_read32_cmac(struct rtw89_dev * rtwdev,u32 addr)1558 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1559 {
1560 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1561 u32 val = readl(rtwpci->mmap + addr);
1562 int count;
1563
1564 for (count = 0; ; count++) {
1565 if (val != RTW89_R32_DEAD)
1566 return val;
1567 if (count >= MAC_REG_POOL_COUNT) {
1568 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1569 return RTW89_R32_DEAD;
1570 }
1571 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1572 val = readl(rtwpci->mmap + addr);
1573 }
1574
1575 return val;
1576 }
1577
rtw89_pci_ops_read8(struct rtw89_dev * rtwdev,u32 addr)1578 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1579 {
1580 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1581 u32 addr32, val32, shift;
1582
1583 if (!ACCESS_CMAC(addr))
1584 return readb(rtwpci->mmap + addr);
1585
1586 addr32 = addr & ~0x3;
1587 shift = (addr & 0x3) * 8;
1588 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1589 return val32 >> shift;
1590 }
1591
rtw89_pci_ops_read16(struct rtw89_dev * rtwdev,u32 addr)1592 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1593 {
1594 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1595 u32 addr32, val32, shift;
1596
1597 if (!ACCESS_CMAC(addr))
1598 return readw(rtwpci->mmap + addr);
1599
1600 addr32 = addr & ~0x3;
1601 shift = (addr & 0x3) * 8;
1602 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1603 return val32 >> shift;
1604 }
1605
rtw89_pci_ops_read32(struct rtw89_dev * rtwdev,u32 addr)1606 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1607 {
1608 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1609
1610 if (!ACCESS_CMAC(addr))
1611 return readl(rtwpci->mmap + addr);
1612
1613 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1614 }
1615
rtw89_pci_ops_write8(struct rtw89_dev * rtwdev,u32 addr,u8 data)1616 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1617 {
1618 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1619
1620 writeb(data, rtwpci->mmap + addr);
1621 }
1622
rtw89_pci_ops_write16(struct rtw89_dev * rtwdev,u32 addr,u16 data)1623 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1624 {
1625 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1626
1627 writew(data, rtwpci->mmap + addr);
1628 }
1629
rtw89_pci_ops_write32(struct rtw89_dev * rtwdev,u32 addr,u32 data)1630 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1631 {
1632 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1633
1634 writel(data, rtwpci->mmap + addr);
1635 }
1636
rtw89_pci_ctrl_dma_trx(struct rtw89_dev * rtwdev,bool enable)1637 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1638 {
1639 const struct rtw89_pci_info *info = rtwdev->pci_info;
1640
1641 if (enable)
1642 rtw89_write32_set(rtwdev, info->init_cfg_reg,
1643 info->rxhci_en_bit | info->txhci_en_bit);
1644 else
1645 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1646 info->rxhci_en_bit | info->txhci_en_bit);
1647 }
1648
rtw89_pci_ctrl_dma_io(struct rtw89_dev * rtwdev,bool enable)1649 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1650 {
1651 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1652 u32 reg, mask;
1653
1654 if (chip_id == RTL8852C) {
1655 reg = R_AX_HAXI_INIT_CFG1;
1656 mask = B_AX_STOP_AXI_MST;
1657 } else {
1658 reg = R_AX_PCIE_DMA_STOP1;
1659 mask = B_AX_STOP_PCIEIO;
1660 }
1661
1662 if (enable)
1663 rtw89_write32_clr(rtwdev, reg, mask);
1664 else
1665 rtw89_write32_set(rtwdev, reg, mask);
1666 }
1667
rtw89_pci_ctrl_dma_all(struct rtw89_dev * rtwdev,bool enable)1668 static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1669 {
1670 rtw89_pci_ctrl_dma_io(rtwdev, enable);
1671 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1672 }
1673
rtw89_pci_check_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 rw_bit)1674 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1675 {
1676 u16 val;
1677
1678 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1679
1680 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1681 switch (speed) {
1682 case PCIE_PHY_GEN1:
1683 if (addr < 0x20)
1684 val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1685 else
1686 val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1687 break;
1688 case PCIE_PHY_GEN2:
1689 if (addr < 0x20)
1690 val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1691 else
1692 val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1693 break;
1694 default:
1695 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1696 return -EINVAL;
1697 }
1698 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
1699 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
1700
1701 return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1702 false, rtwdev, R_AX_MDIO_CFG);
1703 }
1704
1705 static int
rtw89_read16_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 * val)1706 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1707 {
1708 int ret;
1709
1710 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1711 if (ret) {
1712 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1713 return ret;
1714 }
1715 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1716
1717 return 0;
1718 }
1719
1720 static int
rtw89_write16_mdio(struct rtw89_dev * rtwdev,u8 addr,u16 data,u8 speed)1721 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1722 {
1723 int ret;
1724
1725 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1726 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1727 if (ret) {
1728 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1729 return ret;
1730 }
1731
1732 return 0;
1733 }
1734
1735 static int
rtw89_write16_mdio_mask(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u16 data,u8 speed)1736 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1737 {
1738 u32 shift;
1739 int ret;
1740 u16 val;
1741
1742 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1743 if (ret)
1744 return ret;
1745
1746 shift = __ffs(mask);
1747 val &= ~mask;
1748 val |= ((data << shift) & mask);
1749
1750 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
1751 if (ret)
1752 return ret;
1753
1754 return 0;
1755 }
1756
rtw89_write16_mdio_set(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)1757 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1758 {
1759 int ret;
1760 u16 val;
1761
1762 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1763 if (ret)
1764 return ret;
1765 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
1766 if (ret)
1767 return ret;
1768
1769 return 0;
1770 }
1771
rtw89_write16_mdio_clr(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)1772 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1773 {
1774 int ret;
1775 u16 val;
1776
1777 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1778 if (ret)
1779 return ret;
1780 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
1781 if (ret)
1782 return ret;
1783
1784 return 0;
1785 }
1786
rtw89_pci_write_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 data)1787 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1788 u8 data)
1789 {
1790 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1791 struct pci_dev *pdev = rtwpci->pdev;
1792
1793 return pci_write_config_byte(pdev, addr, data);
1794 }
1795
rtw89_pci_read_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 * value)1796 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1797 u8 *value)
1798 {
1799 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1800 struct pci_dev *pdev = rtwpci->pdev;
1801
1802 return pci_read_config_byte(pdev, addr, value);
1803 }
1804
rtw89_pci_config_byte_set(struct rtw89_dev * rtwdev,u16 addr,u8 bit)1805 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
1806 u8 bit)
1807 {
1808 u8 value;
1809 int ret;
1810
1811 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1812 if (ret)
1813 return ret;
1814
1815 value |= bit;
1816 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1817
1818 return ret;
1819 }
1820
rtw89_pci_config_byte_clr(struct rtw89_dev * rtwdev,u16 addr,u8 bit)1821 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
1822 u8 bit)
1823 {
1824 u8 value;
1825 int ret;
1826
1827 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1828 if (ret)
1829 return ret;
1830
1831 value &= ~bit;
1832 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1833
1834 return ret;
1835 }
1836
1837 static int
__get_target(struct rtw89_dev * rtwdev,u16 * target,enum rtw89_pcie_phy phy_rate)1838 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
1839 {
1840 u16 val, tar;
1841 int ret;
1842
1843 /* Enable counter */
1844 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
1845 if (ret)
1846 return ret;
1847 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1848 phy_rate);
1849 if (ret)
1850 return ret;
1851 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
1852 phy_rate);
1853 if (ret)
1854 return ret;
1855
1856 fsleep(300);
1857
1858 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
1859 if (ret)
1860 return ret;
1861 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1862 phy_rate);
1863 if (ret)
1864 return ret;
1865
1866 tar = tar & 0x0FFF;
1867 if (tar == 0 || tar == 0x0FFF) {
1868 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
1869 return -EINVAL;
1870 }
1871
1872 *target = tar;
1873
1874 return 0;
1875 }
1876
rtw89_pci_autok_x(struct rtw89_dev * rtwdev)1877 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
1878 {
1879 int ret;
1880
1881 if (rtwdev->chip->chip_id != RTL8852B)
1882 return 0;
1883
1884 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
1885 PCIE_AUTOK_4, PCIE_PHY_GEN1);
1886 return ret;
1887 }
1888
rtw89_pci_auto_refclk_cal(struct rtw89_dev * rtwdev,bool autook_en)1889 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
1890 {
1891 enum rtw89_pcie_phy phy_rate;
1892 u16 val16, mgn_set, div_set, tar;
1893 u8 val8, bdr_ori;
1894 bool l1_flag = false;
1895 int ret = 0;
1896
1897 if (rtwdev->chip->chip_id != RTL8852B)
1898 return 0;
1899
1900 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
1901 if (ret) {
1902 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
1903 RTW89_PCIE_PHY_RATE);
1904 return ret;
1905 }
1906
1907 if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
1908 phy_rate = PCIE_PHY_GEN1;
1909 } else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
1910 phy_rate = PCIE_PHY_GEN2;
1911 } else {
1912 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
1913 return -EOPNOTSUPP;
1914 }
1915 /* Disable L1BD */
1916 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
1917 if (ret) {
1918 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
1919 return ret;
1920 }
1921
1922 if (bdr_ori & RTW89_PCIE_BIT_L1) {
1923 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
1924 bdr_ori & ~RTW89_PCIE_BIT_L1);
1925 if (ret) {
1926 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
1927 RTW89_PCIE_L1_CTRL);
1928 return ret;
1929 }
1930 l1_flag = true;
1931 }
1932
1933 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
1934 if (ret) {
1935 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
1936 goto end;
1937 }
1938
1939 if (val16 & B_AX_CALIB_EN) {
1940 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
1941 val16 & ~B_AX_CALIB_EN, phy_rate);
1942 if (ret) {
1943 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1944 goto end;
1945 }
1946 }
1947
1948 if (!autook_en)
1949 goto end;
1950 /* Set div */
1951 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
1952 if (ret) {
1953 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1954 goto end;
1955 }
1956
1957 /* Obtain div and margin */
1958 ret = __get_target(rtwdev, &tar, phy_rate);
1959 if (ret) {
1960 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
1961 goto end;
1962 }
1963
1964 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
1965
1966 if (mgn_set >= 128) {
1967 div_set = 0x0003;
1968 mgn_set = 0x000F;
1969 } else if (mgn_set >= 64) {
1970 div_set = 0x0003;
1971 mgn_set >>= 3;
1972 } else if (mgn_set >= 32) {
1973 div_set = 0x0002;
1974 mgn_set >>= 2;
1975 } else if (mgn_set >= 16) {
1976 div_set = 0x0001;
1977 mgn_set >>= 1;
1978 } else if (mgn_set == 0) {
1979 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
1980 goto end;
1981 } else {
1982 div_set = 0x0000;
1983 }
1984
1985 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
1986 if (ret) {
1987 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
1988 goto end;
1989 }
1990
1991 val16 |= u16_encode_bits(div_set, B_AX_DIV);
1992
1993 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
1994 if (ret) {
1995 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1996 goto end;
1997 }
1998
1999 ret = __get_target(rtwdev, &tar, phy_rate);
2000 if (ret) {
2001 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2002 goto end;
2003 }
2004
2005 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2006 tar, div_set, mgn_set);
2007 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2008 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2009 if (ret) {
2010 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2011 goto end;
2012 }
2013
2014 /* Enable function */
2015 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2016 if (ret) {
2017 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2018 goto end;
2019 }
2020
2021 /* CLK delay = 0 */
2022 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2023 PCIE_CLKDLY_HW_0);
2024
2025 end:
2026 /* Set L1BD to ori */
2027 if (l1_flag) {
2028 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2029 bdr_ori);
2030 if (ret) {
2031 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2032 RTW89_PCIE_L1_CTRL);
2033 return ret;
2034 }
2035 }
2036
2037 return ret;
2038 }
2039
rtw89_pci_deglitch_setting(struct rtw89_dev * rtwdev)2040 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2041 {
2042 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2043 int ret;
2044
2045 if (chip_id == RTL8852A) {
2046 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2047 PCIE_PHY_GEN1);
2048 if (ret)
2049 return ret;
2050 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2051 PCIE_PHY_GEN2);
2052 if (ret)
2053 return ret;
2054 } else if (chip_id == RTL8852C) {
2055 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2056 B_AX_DEGLITCH);
2057 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2058 B_AX_DEGLITCH);
2059 }
2060
2061 return 0;
2062 }
2063
rtw89_pci_rxdma_prefth(struct rtw89_dev * rtwdev)2064 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2065 {
2066 if (rtwdev->chip->chip_id != RTL8852A)
2067 return;
2068
2069 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2070 }
2071
rtw89_pci_l1off_pwroff(struct rtw89_dev * rtwdev)2072 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2073 {
2074 if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B)
2075 return;
2076
2077 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2078 }
2079
rtw89_pci_l2_rxen_lat(struct rtw89_dev * rtwdev)2080 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2081 {
2082 int ret;
2083
2084 if (rtwdev->chip->chip_id != RTL8852A)
2085 return 0;
2086
2087 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2088 PCIE_PHY_GEN1);
2089 if (ret)
2090 return ret;
2091
2092 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2093 PCIE_PHY_GEN2);
2094 if (ret)
2095 return ret;
2096
2097 return 0;
2098 }
2099
rtw89_pci_aphy_pwrcut(struct rtw89_dev * rtwdev)2100 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2101 {
2102 if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B)
2103 return;
2104
2105 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2106 }
2107
rtw89_pci_hci_ldo(struct rtw89_dev * rtwdev)2108 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2109 {
2110 if (rtwdev->chip->chip_id == RTL8852A ||
2111 rtwdev->chip->chip_id == RTL8852B) {
2112 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2113 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2114 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2115 B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2116 } else if (rtwdev->chip->chip_id == RTL8852C) {
2117 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2118 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2119 }
2120 }
2121
rtw89_pci_dphy_delay(struct rtw89_dev * rtwdev)2122 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2123 {
2124 if (rtwdev->chip->chip_id != RTL8852B)
2125 return 0;
2126
2127 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2128 PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2129 }
2130
rtw89_pci_power_wake(struct rtw89_dev * rtwdev,bool pwr_up)2131 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2132 {
2133 if (pwr_up)
2134 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2135 else
2136 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2137 }
2138
rtw89_pci_autoload_hang(struct rtw89_dev * rtwdev)2139 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2140 {
2141 if (rtwdev->chip->chip_id != RTL8852C)
2142 return;
2143
2144 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2145 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2146 }
2147
rtw89_pci_l12_vmain(struct rtw89_dev * rtwdev)2148 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2149 {
2150 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2151 return;
2152
2153 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2154 }
2155
rtw89_pci_gen2_force_ib(struct rtw89_dev * rtwdev)2156 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2157 {
2158 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2159 return;
2160
2161 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2162 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2163 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2164 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2165 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2166 }
2167
rtw89_pci_l1_ent_lat(struct rtw89_dev * rtwdev)2168 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2169 {
2170 if (rtwdev->chip->chip_id != RTL8852C)
2171 return;
2172
2173 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2174 }
2175
rtw89_pci_wd_exit_l1(struct rtw89_dev * rtwdev)2176 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2177 {
2178 if (rtwdev->chip->chip_id != RTL8852C)
2179 return;
2180
2181 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2182 }
2183
rtw89_pci_set_sic(struct rtw89_dev * rtwdev)2184 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2185 {
2186 if (rtwdev->chip->chip_id == RTL8852C)
2187 return;
2188
2189 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2190 B_AX_SIC_EN_FORCE_CLKREQ);
2191 }
2192
rtw89_pci_set_lbc(struct rtw89_dev * rtwdev)2193 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2194 {
2195 const struct rtw89_pci_info *info = rtwdev->pci_info;
2196 u32 lbc;
2197
2198 if (rtwdev->chip->chip_id == RTL8852C)
2199 return;
2200
2201 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2202 if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2203 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2204 lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2205 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2206 } else {
2207 lbc &= ~B_AX_LBC_EN;
2208 }
2209 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2210 }
2211
rtw89_pci_set_io_rcy(struct rtw89_dev * rtwdev)2212 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2213 {
2214 const struct rtw89_pci_info *info = rtwdev->pci_info;
2215 u32 val32;
2216
2217 if (rtwdev->chip->chip_id != RTL8852C)
2218 return;
2219
2220 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2221 val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2222 info->io_rcy_tmr);
2223 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2224 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2225 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2226
2227 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2228 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2229 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2230 } else {
2231 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2232 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2233 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2234 }
2235
2236 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2237 }
2238
rtw89_pci_set_dbg(struct rtw89_dev * rtwdev)2239 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2240 {
2241 if (rtwdev->chip->chip_id == RTL8852C)
2242 return;
2243
2244 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2245 B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2246
2247 if (rtwdev->chip->chip_id == RTL8852A)
2248 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2249 B_AX_EN_CHKDSC_NO_RX_STUCK);
2250 }
2251
rtw89_pci_set_keep_reg(struct rtw89_dev * rtwdev)2252 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2253 {
2254 if (rtwdev->chip->chip_id == RTL8852C)
2255 return;
2256
2257 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2258 B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2259 }
2260
rtw89_pci_clr_idx_all(struct rtw89_dev * rtwdev)2261 static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)
2262 {
2263 const struct rtw89_pci_info *info = rtwdev->pci_info;
2264 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2265 u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2266 B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2267 B_AX_CLR_CH12_IDX;
2268 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2269 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2270
2271 if (chip_id == RTL8852A || chip_id == RTL8852C)
2272 val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2273 B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2274 /* clear DMA indexes */
2275 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2276 if (chip_id == RTL8852A || chip_id == RTL8852C)
2277 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2278 B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2279 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2280 B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2281 }
2282
rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev * rtwdev)2283 static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2284 {
2285 const struct rtw89_pci_info *info = rtwdev->pci_info;
2286 u32 ret, check, dma_busy;
2287 u32 dma_busy1 = info->dma_busy1.addr;
2288 u32 dma_busy2 = info->dma_busy2_reg;
2289
2290 check = info->dma_busy1.mask;
2291
2292 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2293 10, 100, false, rtwdev, dma_busy1);
2294 if (ret)
2295 return ret;
2296
2297 if (!dma_busy2)
2298 return 0;
2299
2300 check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2301
2302 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2303 10, 100, false, rtwdev, dma_busy2);
2304 if (ret)
2305 return ret;
2306
2307 return 0;
2308 }
2309
rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev * rtwdev)2310 static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2311 {
2312 const struct rtw89_pci_info *info = rtwdev->pci_info;
2313 u32 ret, check, dma_busy;
2314 u32 dma_busy3 = info->dma_busy3_reg;
2315
2316 check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2317
2318 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2319 10, 100, false, rtwdev, dma_busy3);
2320 if (ret)
2321 return ret;
2322
2323 return 0;
2324 }
2325
rtw89_pci_poll_dma_all_idle(struct rtw89_dev * rtwdev)2326 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2327 {
2328 u32 ret;
2329
2330 ret = rtw89_poll_txdma_ch_idle_pcie(rtwdev);
2331 if (ret) {
2332 rtw89_err(rtwdev, "txdma ch busy\n");
2333 return ret;
2334 }
2335
2336 ret = rtw89_poll_rxdma_ch_idle_pcie(rtwdev);
2337 if (ret) {
2338 rtw89_err(rtwdev, "rxdma ch busy\n");
2339 return ret;
2340 }
2341
2342 return 0;
2343 }
2344
rtw89_pci_mode_op(struct rtw89_dev * rtwdev)2345 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2346 {
2347 const struct rtw89_pci_info *info = rtwdev->pci_info;
2348 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2349 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2350 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2351 enum mac_ax_tag_mode tag_mode = info->tag_mode;
2352 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2353 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2354 enum mac_ax_tx_burst tx_burst = info->tx_burst;
2355 enum mac_ax_rx_burst rx_burst = info->rx_burst;
2356 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2357 u8 cv = rtwdev->hal.cv;
2358 u32 val32;
2359
2360 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2361 if (chip_id == RTL8852A && cv == CHIP_CBV)
2362 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2363 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2364 if (chip_id == RTL8852A || chip_id == RTL8852B)
2365 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2366 }
2367
2368 if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2369 if (chip_id == RTL8852A && cv == CHIP_CBV)
2370 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2371 } else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2372 if (chip_id == RTL8852A || chip_id == RTL8852B)
2373 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2374 }
2375
2376 if (rxbd_mode == MAC_AX_RXBD_PKT) {
2377 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2378 } else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2379 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2380
2381 if (chip_id == RTL8852A || chip_id == RTL8852B)
2382 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2383 B_AX_PCIE_RX_APPLEN_MASK, 0);
2384 }
2385
2386 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2387 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2388 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2389 } else if (chip_id == RTL8852C) {
2390 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2391 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2392 }
2393
2394 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2395 if (tag_mode == MAC_AX_TAG_SGL) {
2396 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2397 ~B_AX_LATENCY_CONTROL;
2398 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2399 } else if (tag_mode == MAC_AX_TAG_MULTI) {
2400 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2401 B_AX_LATENCY_CONTROL;
2402 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2403 }
2404 }
2405
2406 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2407 info->multi_tag_num);
2408
2409 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2410 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2411 wd_dma_idle_intvl);
2412 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2413 wd_dma_act_intvl);
2414 } else if (chip_id == RTL8852C) {
2415 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2416 wd_dma_idle_intvl);
2417 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2418 wd_dma_act_intvl);
2419 }
2420
2421 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2422 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2423 B_AX_HOST_ADDR_INFO_8B_SEL);
2424 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2425 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2426 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2427 B_AX_HOST_ADDR_INFO_8B_SEL);
2428 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2429 }
2430
2431 return 0;
2432 }
2433
rtw89_pci_ops_deinit(struct rtw89_dev * rtwdev)2434 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2435 {
2436 const struct rtw89_pci_info *info = rtwdev->pci_info;
2437
2438 if (rtwdev->chip->chip_id == RTL8852A) {
2439 /* ltr sw trigger */
2440 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2441 }
2442 info->ltr_set(rtwdev, false);
2443 rtw89_pci_ctrl_dma_all(rtwdev, false);
2444 rtw89_pci_clr_idx_all(rtwdev);
2445
2446 return 0;
2447 }
2448
rtw89_pci_ops_mac_pre_init(struct rtw89_dev * rtwdev)2449 static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
2450 {
2451 const struct rtw89_pci_info *info = rtwdev->pci_info;
2452 int ret;
2453
2454 rtw89_pci_rxdma_prefth(rtwdev);
2455 rtw89_pci_l1off_pwroff(rtwdev);
2456 rtw89_pci_deglitch_setting(rtwdev);
2457 ret = rtw89_pci_l2_rxen_lat(rtwdev);
2458 if (ret) {
2459 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2460 return ret;
2461 }
2462
2463 rtw89_pci_aphy_pwrcut(rtwdev);
2464 rtw89_pci_hci_ldo(rtwdev);
2465 rtw89_pci_dphy_delay(rtwdev);
2466
2467 ret = rtw89_pci_autok_x(rtwdev);
2468 if (ret) {
2469 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2470 return ret;
2471 }
2472
2473 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2474 if (ret) {
2475 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2476 return ret;
2477 }
2478
2479 rtw89_pci_power_wake(rtwdev, true);
2480 rtw89_pci_autoload_hang(rtwdev);
2481 rtw89_pci_l12_vmain(rtwdev);
2482 rtw89_pci_gen2_force_ib(rtwdev);
2483 rtw89_pci_l1_ent_lat(rtwdev);
2484 rtw89_pci_wd_exit_l1(rtwdev);
2485 rtw89_pci_set_sic(rtwdev);
2486 rtw89_pci_set_lbc(rtwdev);
2487 rtw89_pci_set_io_rcy(rtwdev);
2488 rtw89_pci_set_dbg(rtwdev);
2489 rtw89_pci_set_keep_reg(rtwdev);
2490
2491 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2492
2493 /* stop DMA activities */
2494 rtw89_pci_ctrl_dma_all(rtwdev, false);
2495
2496 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2497 if (ret) {
2498 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2499 return ret;
2500 }
2501
2502 rtw89_pci_clr_idx_all(rtwdev);
2503 rtw89_pci_mode_op(rtwdev);
2504
2505 /* fill TRX BD indexes */
2506 rtw89_pci_ops_reset(rtwdev);
2507
2508 ret = rtw89_pci_rst_bdram_pcie(rtwdev);
2509 if (ret) {
2510 rtw89_warn(rtwdev, "reset bdram busy\n");
2511 return ret;
2512 }
2513
2514 /* disable all channels except to FW CMD channel to download firmware */
2515 rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false);
2516 rtw89_write32_clr(rtwdev, info->dma_stop1.addr, B_AX_STOP_CH12);
2517
2518 /* start DMA activities */
2519 rtw89_pci_ctrl_dma_all(rtwdev, true);
2520
2521 return 0;
2522 }
2523
rtw89_pci_ltr_set(struct rtw89_dev * rtwdev,bool en)2524 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2525 {
2526 u32 val;
2527
2528 if (!en)
2529 return 0;
2530
2531 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2532 if (rtw89_pci_ltr_is_err_reg_val(val))
2533 return -EINVAL;
2534 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2535 if (rtw89_pci_ltr_is_err_reg_val(val))
2536 return -EINVAL;
2537 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2538 if (rtw89_pci_ltr_is_err_reg_val(val))
2539 return -EINVAL;
2540 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2541 if (rtw89_pci_ltr_is_err_reg_val(val))
2542 return -EINVAL;
2543
2544 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2545 B_AX_LTR_WD_NOEMP_CHK);
2546 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2547 PCI_LTR_SPC_500US);
2548 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2549 PCI_LTR_IDLE_TIMER_3_2MS);
2550 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2551 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2552 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2553 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2554
2555 return 0;
2556 }
2557 EXPORT_SYMBOL(rtw89_pci_ltr_set);
2558
rtw89_pci_ltr_set_v1(struct rtw89_dev * rtwdev,bool en)2559 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2560 {
2561 u32 dec_ctrl;
2562 u32 val32;
2563
2564 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2565 if (rtw89_pci_ltr_is_err_reg_val(val32))
2566 return -EINVAL;
2567 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2568 if (rtw89_pci_ltr_is_err_reg_val(val32))
2569 return -EINVAL;
2570 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2571 if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
2572 return -EINVAL;
2573 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2574 if (rtw89_pci_ltr_is_err_reg_val(val32))
2575 return -EINVAL;
2576 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2577 if (rtw89_pci_ltr_is_err_reg_val(val32))
2578 return -EINVAL;
2579
2580 if (!en) {
2581 dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2582 dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2583 B_AX_LTR_REQ_DRV;
2584 } else {
2585 dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2586 }
2587
2588 dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2589 dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2590
2591 if (en)
2592 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2593 B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
2594 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2595 PCI_LTR_IDLE_TIMER_3_2MS);
2596 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2597 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2598 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
2599 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
2600 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
2601
2602 return 0;
2603 }
2604 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
2605
rtw89_pci_ops_mac_post_init(struct rtw89_dev * rtwdev)2606 static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
2607 {
2608 const struct rtw89_pci_info *info = rtwdev->pci_info;
2609 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2610 int ret;
2611
2612 ret = info->ltr_set(rtwdev, true);
2613 if (ret) {
2614 rtw89_err(rtwdev, "pci ltr set fail\n");
2615 return ret;
2616 }
2617 if (chip_id == RTL8852A) {
2618 /* ltr sw trigger */
2619 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
2620 }
2621 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2622 /* ADDR info 8-byte mode */
2623 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2624 B_AX_HOST_ADDR_INFO_8B_SEL);
2625 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2626 }
2627
2628 /* enable DMA for all queues */
2629 rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true);
2630
2631 /* Release PCI IO */
2632 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
2633 B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
2634
2635 return 0;
2636 }
2637
rtw89_pci_claim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2638 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
2639 struct pci_dev *pdev)
2640 {
2641 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2642 int ret;
2643
2644 ret = pci_enable_device(pdev);
2645 if (ret) {
2646 rtw89_err(rtwdev, "failed to enable pci device\n");
2647 return ret;
2648 }
2649
2650 pci_set_master(pdev);
2651 pci_set_drvdata(pdev, rtwdev->hw);
2652
2653 rtwpci->pdev = pdev;
2654
2655 return 0;
2656 }
2657
rtw89_pci_declaim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2658 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
2659 struct pci_dev *pdev)
2660 {
2661 pci_clear_master(pdev);
2662 pci_disable_device(pdev);
2663 }
2664
rtw89_pci_setup_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2665 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
2666 struct pci_dev *pdev)
2667 {
2668 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2669 unsigned long resource_len;
2670 u8 bar_id = 2;
2671 int ret;
2672
2673 ret = pci_request_regions(pdev, KBUILD_MODNAME);
2674 if (ret) {
2675 rtw89_err(rtwdev, "failed to request pci regions\n");
2676 goto err;
2677 }
2678
2679 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2680 if (ret) {
2681 rtw89_err(rtwdev, "failed to set dma mask to 32-bit\n");
2682 goto err_release_regions;
2683 }
2684
2685 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
2686 if (ret) {
2687 rtw89_err(rtwdev, "failed to set consistent dma mask to 32-bit\n");
2688 goto err_release_regions;
2689 }
2690
2691 resource_len = pci_resource_len(pdev, bar_id);
2692 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
2693 if (!rtwpci->mmap) {
2694 rtw89_err(rtwdev, "failed to map pci io\n");
2695 ret = -EIO;
2696 goto err_release_regions;
2697 }
2698
2699 return 0;
2700
2701 err_release_regions:
2702 pci_release_regions(pdev);
2703 err:
2704 return ret;
2705 }
2706
rtw89_pci_clear_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2707 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
2708 struct pci_dev *pdev)
2709 {
2710 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2711
2712 if (rtwpci->mmap) {
2713 pci_iounmap(pdev, rtwpci->mmap);
2714 pci_release_regions(pdev);
2715 }
2716 }
2717
rtw89_pci_free_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)2718 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
2719 struct pci_dev *pdev,
2720 struct rtw89_pci_tx_ring *tx_ring)
2721 {
2722 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2723 u8 *head = wd_ring->head;
2724 dma_addr_t dma = wd_ring->dma;
2725 u32 page_size = wd_ring->page_size;
2726 u32 page_num = wd_ring->page_num;
2727 u32 ring_sz = page_size * page_num;
2728
2729 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2730 wd_ring->head = NULL;
2731 }
2732
rtw89_pci_free_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)2733 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
2734 struct pci_dev *pdev,
2735 struct rtw89_pci_tx_ring *tx_ring)
2736 {
2737 int ring_sz;
2738 u8 *head;
2739 dma_addr_t dma;
2740
2741 head = tx_ring->bd_ring.head;
2742 dma = tx_ring->bd_ring.dma;
2743 ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
2744 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2745
2746 tx_ring->bd_ring.head = NULL;
2747 }
2748
rtw89_pci_free_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2749 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
2750 struct pci_dev *pdev)
2751 {
2752 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2753 const struct rtw89_pci_info *info = rtwdev->pci_info;
2754 struct rtw89_pci_tx_ring *tx_ring;
2755 int i;
2756
2757 for (i = 0; i < RTW89_TXCH_NUM; i++) {
2758 if (info->tx_dma_ch_mask & BIT(i))
2759 continue;
2760 tx_ring = &rtwpci->tx_rings[i];
2761 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2762 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
2763 }
2764 }
2765
rtw89_pci_free_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring)2766 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
2767 struct pci_dev *pdev,
2768 struct rtw89_pci_rx_ring *rx_ring)
2769 {
2770 struct rtw89_pci_rx_info *rx_info;
2771 struct sk_buff *skb;
2772 dma_addr_t dma;
2773 u32 buf_sz;
2774 u8 *head;
2775 int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
2776 int i;
2777
2778 buf_sz = rx_ring->buf_sz;
2779 for (i = 0; i < rx_ring->bd_ring.len; i++) {
2780 skb = rx_ring->buf[i];
2781 if (!skb)
2782 continue;
2783
2784 rx_info = RTW89_PCI_RX_SKB_CB(skb);
2785 dma = rx_info->dma;
2786 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
2787 dev_kfree_skb(skb);
2788 rx_ring->buf[i] = NULL;
2789 }
2790
2791 head = rx_ring->bd_ring.head;
2792 dma = rx_ring->bd_ring.dma;
2793 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2794
2795 rx_ring->bd_ring.head = NULL;
2796 }
2797
rtw89_pci_free_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2798 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
2799 struct pci_dev *pdev)
2800 {
2801 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2802 struct rtw89_pci_rx_ring *rx_ring;
2803 int i;
2804
2805 for (i = 0; i < RTW89_RXCH_NUM; i++) {
2806 rx_ring = &rtwpci->rx_rings[i];
2807 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
2808 }
2809 }
2810
rtw89_pci_free_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2811 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
2812 struct pci_dev *pdev)
2813 {
2814 rtw89_pci_free_rx_rings(rtwdev, pdev);
2815 rtw89_pci_free_tx_rings(rtwdev, pdev);
2816 }
2817
rtw89_pci_init_rx_bd(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb,int buf_sz,u32 idx)2818 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
2819 struct rtw89_pci_rx_ring *rx_ring,
2820 struct sk_buff *skb, int buf_sz, u32 idx)
2821 {
2822 struct rtw89_pci_rx_info *rx_info;
2823 struct rtw89_pci_rx_bd_32 *rx_bd;
2824 dma_addr_t dma;
2825
2826 if (!skb)
2827 return -EINVAL;
2828
2829 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
2830 if (dma_mapping_error(&pdev->dev, dma))
2831 return -EBUSY;
2832
2833 rx_info = RTW89_PCI_RX_SKB_CB(skb);
2834 rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
2835
2836 memset(rx_bd, 0, sizeof(*rx_bd));
2837 rx_bd->buf_size = cpu_to_le16(buf_sz);
2838 rx_bd->dma = cpu_to_le32(dma);
2839 rx_info->dma = dma;
2840
2841 return 0;
2842 }
2843
rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,enum rtw89_tx_channel txch)2844 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
2845 struct pci_dev *pdev,
2846 struct rtw89_pci_tx_ring *tx_ring,
2847 enum rtw89_tx_channel txch)
2848 {
2849 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2850 struct rtw89_pci_tx_wd *txwd;
2851 dma_addr_t dma;
2852 dma_addr_t cur_paddr;
2853 u8 *head;
2854 u8 *cur_vaddr;
2855 u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
2856 u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
2857 u32 ring_sz = page_size * page_num;
2858 u32 page_offset;
2859 int i;
2860
2861 /* FWCMD queue doesn't use txwd as pages */
2862 if (txch == RTW89_TXCH_CH12)
2863 return 0;
2864
2865 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2866 if (!head)
2867 return -ENOMEM;
2868
2869 INIT_LIST_HEAD(&wd_ring->free_pages);
2870 wd_ring->head = head;
2871 wd_ring->dma = dma;
2872 wd_ring->page_size = page_size;
2873 wd_ring->page_num = page_num;
2874
2875 page_offset = 0;
2876 for (i = 0; i < page_num; i++) {
2877 txwd = &wd_ring->pages[i];
2878 cur_paddr = dma + page_offset;
2879 cur_vaddr = head + page_offset;
2880
2881 skb_queue_head_init(&txwd->queue);
2882 INIT_LIST_HEAD(&txwd->list);
2883 txwd->paddr = cur_paddr;
2884 txwd->vaddr = cur_vaddr;
2885 txwd->len = page_size;
2886 txwd->seq = i;
2887 rtw89_pci_enqueue_txwd(tx_ring, txwd);
2888
2889 page_offset += page_size;
2890 }
2891
2892 return 0;
2893 }
2894
rtw89_pci_alloc_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,u32 desc_size,u32 len,enum rtw89_tx_channel txch)2895 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
2896 struct pci_dev *pdev,
2897 struct rtw89_pci_tx_ring *tx_ring,
2898 u32 desc_size, u32 len,
2899 enum rtw89_tx_channel txch)
2900 {
2901 const struct rtw89_pci_ch_dma_addr *txch_addr;
2902 int ring_sz = desc_size * len;
2903 u8 *head;
2904 dma_addr_t dma;
2905 int ret;
2906
2907 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
2908 if (ret) {
2909 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
2910 goto err;
2911 }
2912
2913 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
2914 if (ret) {
2915 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
2916 goto err_free_wd_ring;
2917 }
2918
2919 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2920 if (!head) {
2921 ret = -ENOMEM;
2922 goto err_free_wd_ring;
2923 }
2924
2925 INIT_LIST_HEAD(&tx_ring->busy_pages);
2926 tx_ring->bd_ring.head = head;
2927 tx_ring->bd_ring.dma = dma;
2928 tx_ring->bd_ring.len = len;
2929 tx_ring->bd_ring.desc_size = desc_size;
2930 tx_ring->bd_ring.addr = *txch_addr;
2931 tx_ring->bd_ring.wp = 0;
2932 tx_ring->bd_ring.rp = 0;
2933 tx_ring->txch = txch;
2934
2935 return 0;
2936
2937 err_free_wd_ring:
2938 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2939 err:
2940 return ret;
2941 }
2942
rtw89_pci_alloc_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2943 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
2944 struct pci_dev *pdev)
2945 {
2946 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2947 const struct rtw89_pci_info *info = rtwdev->pci_info;
2948 struct rtw89_pci_tx_ring *tx_ring;
2949 u32 desc_size;
2950 u32 len;
2951 u32 i, tx_allocated;
2952 int ret;
2953
2954 for (i = 0; i < RTW89_TXCH_NUM; i++) {
2955 if (info->tx_dma_ch_mask & BIT(i))
2956 continue;
2957 tx_ring = &rtwpci->tx_rings[i];
2958 desc_size = sizeof(struct rtw89_pci_tx_bd_32);
2959 len = RTW89_PCI_TXBD_NUM_MAX;
2960 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
2961 desc_size, len, i);
2962 if (ret) {
2963 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
2964 goto err_free;
2965 }
2966 }
2967
2968 return 0;
2969
2970 err_free:
2971 tx_allocated = i;
2972 for (i = 0; i < tx_allocated; i++) {
2973 tx_ring = &rtwpci->tx_rings[i];
2974 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
2975 }
2976
2977 return ret;
2978 }
2979
rtw89_pci_alloc_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,u32 desc_size,u32 len,u32 rxch)2980 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
2981 struct pci_dev *pdev,
2982 struct rtw89_pci_rx_ring *rx_ring,
2983 u32 desc_size, u32 len, u32 rxch)
2984 {
2985 const struct rtw89_pci_ch_dma_addr *rxch_addr;
2986 struct sk_buff *skb;
2987 u8 *head;
2988 dma_addr_t dma;
2989 int ring_sz = desc_size * len;
2990 int buf_sz = RTW89_PCI_RX_BUF_SIZE;
2991 int i, allocated;
2992 int ret;
2993
2994 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
2995 if (ret) {
2996 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
2997 return ret;
2998 }
2999
3000 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3001 if (!head) {
3002 ret = -ENOMEM;
3003 goto err;
3004 }
3005
3006 rx_ring->bd_ring.head = head;
3007 rx_ring->bd_ring.dma = dma;
3008 rx_ring->bd_ring.len = len;
3009 rx_ring->bd_ring.desc_size = desc_size;
3010 rx_ring->bd_ring.addr = *rxch_addr;
3011 rx_ring->bd_ring.wp = 0;
3012 rx_ring->bd_ring.rp = 0;
3013 rx_ring->buf_sz = buf_sz;
3014 rx_ring->diliver_skb = NULL;
3015 rx_ring->diliver_desc.ready = false;
3016
3017 for (i = 0; i < len; i++) {
3018 skb = dev_alloc_skb(buf_sz);
3019 if (!skb) {
3020 ret = -ENOMEM;
3021 goto err_free;
3022 }
3023
3024 memset(skb->data, 0, buf_sz);
3025 rx_ring->buf[i] = skb;
3026 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3027 buf_sz, i);
3028 if (ret) {
3029 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3030 dev_kfree_skb_any(skb);
3031 rx_ring->buf[i] = NULL;
3032 goto err_free;
3033 }
3034 }
3035
3036 return 0;
3037
3038 err_free:
3039 allocated = i;
3040 for (i = 0; i < allocated; i++) {
3041 skb = rx_ring->buf[i];
3042 if (!skb)
3043 continue;
3044 dma = *((dma_addr_t *)skb->cb);
3045 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3046 dev_kfree_skb(skb);
3047 rx_ring->buf[i] = NULL;
3048 }
3049
3050 head = rx_ring->bd_ring.head;
3051 dma = rx_ring->bd_ring.dma;
3052 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3053
3054 rx_ring->bd_ring.head = NULL;
3055 err:
3056 return ret;
3057 }
3058
rtw89_pci_alloc_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3059 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3060 struct pci_dev *pdev)
3061 {
3062 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3063 struct rtw89_pci_rx_ring *rx_ring;
3064 u32 desc_size;
3065 u32 len;
3066 int i, rx_allocated;
3067 int ret;
3068
3069 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3070 rx_ring = &rtwpci->rx_rings[i];
3071 desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3072 len = RTW89_PCI_RXBD_NUM_MAX;
3073 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3074 desc_size, len, i);
3075 if (ret) {
3076 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3077 goto err_free;
3078 }
3079 }
3080
3081 return 0;
3082
3083 err_free:
3084 rx_allocated = i;
3085 for (i = 0; i < rx_allocated; i++) {
3086 rx_ring = &rtwpci->rx_rings[i];
3087 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3088 }
3089
3090 return ret;
3091 }
3092
rtw89_pci_alloc_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3093 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3094 struct pci_dev *pdev)
3095 {
3096 int ret;
3097
3098 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3099 if (ret) {
3100 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3101 goto err;
3102 }
3103
3104 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3105 if (ret) {
3106 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3107 goto err_free_tx_rings;
3108 }
3109
3110 return 0;
3111
3112 err_free_tx_rings:
3113 rtw89_pci_free_tx_rings(rtwdev, pdev);
3114 err:
3115 return ret;
3116 }
3117
rtw89_pci_h2c_init(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)3118 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3119 struct rtw89_pci *rtwpci)
3120 {
3121 skb_queue_head_init(&rtwpci->h2c_queue);
3122 skb_queue_head_init(&rtwpci->h2c_release_queue);
3123 }
3124
rtw89_pci_setup_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3125 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3126 struct pci_dev *pdev)
3127 {
3128 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3129 int ret;
3130
3131 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3132 if (ret) {
3133 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3134 goto err;
3135 }
3136
3137 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3138 if (ret) {
3139 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3140 goto err_pci_unmap;
3141 }
3142
3143 rtw89_pci_h2c_init(rtwdev, rtwpci);
3144
3145 spin_lock_init(&rtwpci->irq_lock);
3146 spin_lock_init(&rtwpci->trx_lock);
3147
3148 return 0;
3149
3150 err_pci_unmap:
3151 rtw89_pci_clear_mapping(rtwdev, pdev);
3152 err:
3153 return ret;
3154 }
3155
rtw89_pci_clear_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3156 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3157 struct pci_dev *pdev)
3158 {
3159 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3160
3161 rtw89_pci_free_trx_rings(rtwdev, pdev);
3162 rtw89_pci_clear_mapping(rtwdev, pdev);
3163 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3164 skb_queue_len(&rtwpci->h2c_queue), true);
3165 }
3166
rtw89_pci_config_intr_mask(struct rtw89_dev * rtwdev)3167 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3168 {
3169 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3170
3171 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3172
3173 if (rtwpci->under_recovery) {
3174 rtwpci->intrs[0] = B_AX_HS0ISR_IND_INT_EN;
3175 rtwpci->intrs[1] = 0;
3176 } else {
3177 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3178 B_AX_RXDMA_INT_EN |
3179 B_AX_RXP1DMA_INT_EN |
3180 B_AX_RPQDMA_INT_EN |
3181 B_AX_RXDMA_STUCK_INT_EN |
3182 B_AX_RDU_INT_EN |
3183 B_AX_RPQBD_FULL_INT_EN |
3184 B_AX_HS0ISR_IND_INT_EN;
3185
3186 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3187 }
3188 }
3189 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3190
rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev * rtwdev)3191 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3192 {
3193 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3194
3195 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3196 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3197 rtwpci->intrs[0] = 0;
3198 rtwpci->intrs[1] = 0;
3199 }
3200
rtw89_pci_default_intr_mask_v1(struct rtw89_dev * rtwdev)3201 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3202 {
3203 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3204
3205 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3206 B_AX_HS1ISR_IND_INT_EN |
3207 B_AX_HS0ISR_IND_INT_EN;
3208 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3209 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3210 B_AX_RXDMA_INT_EN |
3211 B_AX_RXP1DMA_INT_EN |
3212 B_AX_RPQDMA_INT_EN |
3213 B_AX_RXDMA_STUCK_INT_EN |
3214 B_AX_RDU_INT_EN |
3215 B_AX_RPQBD_FULL_INT_EN;
3216 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3217 }
3218
rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev * rtwdev)3219 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3220 {
3221 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3222
3223 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3224 B_AX_HS0ISR_IND_INT_EN;
3225 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3226 rtwpci->intrs[0] = 0;
3227 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3228 }
3229
rtw89_pci_config_intr_mask_v1(struct rtw89_dev * rtwdev)3230 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3231 {
3232 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3233
3234 if (rtwpci->under_recovery)
3235 rtw89_pci_recovery_intr_mask_v1(rtwdev);
3236 else if (rtwpci->low_power)
3237 rtw89_pci_low_power_intr_mask_v1(rtwdev);
3238 else
3239 rtw89_pci_default_intr_mask_v1(rtwdev);
3240 }
3241 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3242
rtw89_pci_request_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3243 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3244 struct pci_dev *pdev)
3245 {
3246 unsigned long flags = 0;
3247 int ret;
3248
3249 flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI;
3250 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3251 if (ret < 0) {
3252 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3253 goto err;
3254 }
3255
3256 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3257 rtw89_pci_interrupt_handler,
3258 rtw89_pci_interrupt_threadfn,
3259 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3260 if (ret) {
3261 rtw89_err(rtwdev, "failed to request threaded irq\n");
3262 goto err_free_vector;
3263 }
3264
3265 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3266
3267 return 0;
3268
3269 err_free_vector:
3270 pci_free_irq_vectors(pdev);
3271 err:
3272 return ret;
3273 }
3274
rtw89_pci_free_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3275 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3276 struct pci_dev *pdev)
3277 {
3278 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3279 pci_free_irq_vectors(pdev);
3280 }
3281
gray_code_to_bin(u16 gray_code,u32 bit_num)3282 static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
3283 {
3284 u16 bin = 0, gray_bit;
3285 u32 bit_idx;
3286
3287 for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
3288 gray_bit = (gray_code >> bit_idx) & 0x1;
3289 if (bit_num - bit_idx > 1)
3290 gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
3291 bin |= (gray_bit << bit_idx);
3292 }
3293
3294 return bin;
3295 }
3296
rtw89_pci_filter_out(struct rtw89_dev * rtwdev)3297 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3298 {
3299 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3300 struct pci_dev *pdev = rtwpci->pdev;
3301 u16 val16, filter_out_val;
3302 u32 val, phy_offset;
3303 int ret;
3304
3305 if (rtwdev->chip->chip_id != RTL8852C)
3306 return 0;
3307
3308 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3309 if (val == B_AX_ASPM_CTRL_L1)
3310 return 0;
3311
3312 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
3313 if (ret)
3314 return ret;
3315
3316 val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3317 if (val == RTW89_PCIE_GEN1_SPEED) {
3318 phy_offset = R_RAC_DIRECT_OFFSET_G1;
3319 } else if (val == RTW89_PCIE_GEN2_SPEED) {
3320 phy_offset = R_RAC_DIRECT_OFFSET_G2;
3321 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3322 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3323 val16 | B_PCIE_BIT_PINOUT_DIS);
3324 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3325 val16 & ~B_PCIE_BIT_RD_SEL);
3326
3327 val16 = rtw89_read16_mask(rtwdev,
3328 phy_offset + RAC_ANA1F * RAC_MULT,
3329 FILTER_OUT_EQ_MASK);
3330 val16 = gray_code_to_bin(val16, hweight16(val16));
3331 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3332 RAC_MULT);
3333 filter_out_val &= ~REG_FILTER_OUT_MASK;
3334 filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3335
3336 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3337 filter_out_val);
3338 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3339 B_BAC_EQ_SEL);
3340 rtw89_write16_set(rtwdev,
3341 R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3342 B_PCIE_BIT_PSAVE);
3343 } else {
3344 return -EOPNOTSUPP;
3345 }
3346 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3347 B_PCIE_BIT_PSAVE);
3348
3349 return 0;
3350 }
3351
rtw89_pci_clkreq_set(struct rtw89_dev * rtwdev,bool enable)3352 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3353 {
3354 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3355 int ret;
3356
3357 if (rtw89_pci_disable_clkreq)
3358 return;
3359
3360 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3361 PCIE_CLKDLY_HW_30US);
3362 if (ret)
3363 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3364
3365 if (chip_id == RTL8852A) {
3366 if (enable)
3367 ret = rtw89_pci_config_byte_set(rtwdev,
3368 RTW89_PCIE_L1_CTRL,
3369 RTW89_PCIE_BIT_CLK);
3370 else
3371 ret = rtw89_pci_config_byte_clr(rtwdev,
3372 RTW89_PCIE_L1_CTRL,
3373 RTW89_PCIE_BIT_CLK);
3374 if (ret)
3375 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3376 enable ? "set" : "unset", ret);
3377 } else if (chip_id == RTL8852C) {
3378 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3379 B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3380 if (enable)
3381 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3382 B_AX_CLK_REQ_N);
3383 else
3384 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3385 B_AX_CLK_REQ_N);
3386 }
3387 }
3388
rtw89_pci_aspm_set(struct rtw89_dev * rtwdev,bool enable)3389 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3390 {
3391 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3392 u8 value = 0;
3393 int ret;
3394
3395 if (rtw89_pci_disable_aspm_l1)
3396 return;
3397
3398 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3399 if (ret)
3400 rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3401
3402 value &= ~(RTW89_L1DLY_MASK | RTW89_L0DLY_MASK);
3403 value |= FIELD_PREP(RTW89_L1DLY_MASK, PCIE_L1DLY_16US) |
3404 FIELD_PREP(RTW89_L0DLY_MASK, PCIE_L0SDLY_4US);
3405
3406 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3407 if (ret)
3408 rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3409
3410 if (chip_id == RTL8852A || chip_id == RTL8852B) {
3411 if (enable)
3412 ret = rtw89_pci_config_byte_set(rtwdev,
3413 RTW89_PCIE_L1_CTRL,
3414 RTW89_PCIE_BIT_L1);
3415 else
3416 ret = rtw89_pci_config_byte_clr(rtwdev,
3417 RTW89_PCIE_L1_CTRL,
3418 RTW89_PCIE_BIT_L1);
3419 } else if (chip_id == RTL8852C) {
3420 if (enable)
3421 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3422 B_AX_ASPM_CTRL_L1);
3423 else
3424 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3425 B_AX_ASPM_CTRL_L1);
3426 }
3427 if (ret)
3428 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3429 enable ? "set" : "unset", ret);
3430 }
3431
rtw89_pci_recalc_int_mit(struct rtw89_dev * rtwdev)3432 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3433 {
3434 struct rtw89_traffic_stats *stats = &rtwdev->stats;
3435 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3436 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3437 u32 val = 0;
3438
3439 if (!rtwdev->scanning &&
3440 (tx_tfc_lv >= RTW89_TFC_HIGH || rx_tfc_lv >= RTW89_TFC_HIGH))
3441 val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3442 FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3443 FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3444 FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3445
3446 rtw89_write32(rtwdev, R_AX_INT_MIT_RX, val);
3447 }
3448
rtw89_pci_link_cfg(struct rtw89_dev * rtwdev)3449 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
3450 {
3451 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3452 struct pci_dev *pdev = rtwpci->pdev;
3453 u16 link_ctrl;
3454 int ret;
3455
3456 /* Though there is standard PCIE configuration space to set the
3457 * link control register, but by Realtek's design, driver should
3458 * check if host supports CLKREQ/ASPM to enable the HW module.
3459 *
3460 * These functions are implemented by two HW modules associated,
3461 * one is responsible to access PCIE configuration space to
3462 * follow the host settings, and another is in charge of doing
3463 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
3464 * the host does not support it, and due to some reasons or wrong
3465 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
3466 * loss if HW misbehaves on the link.
3467 *
3468 * Hence it's designed that driver should first check the PCIE
3469 * configuration space is sync'ed and enabled, then driver can turn
3470 * on the other module that is actually working on the mechanism.
3471 */
3472 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
3473 if (ret) {
3474 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
3475 return;
3476 }
3477
3478 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
3479 rtw89_pci_clkreq_set(rtwdev, true);
3480
3481 if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
3482 rtw89_pci_aspm_set(rtwdev, true);
3483 }
3484
rtw89_pci_l1ss_set(struct rtw89_dev * rtwdev,bool enable)3485 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
3486 {
3487 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3488 int ret;
3489
3490 if (chip_id == RTL8852A || chip_id == RTL8852B) {
3491 if (enable)
3492 ret = rtw89_pci_config_byte_set(rtwdev,
3493 RTW89_PCIE_TIMER_CTRL,
3494 RTW89_PCIE_BIT_L1SUB);
3495 else
3496 ret = rtw89_pci_config_byte_clr(rtwdev,
3497 RTW89_PCIE_TIMER_CTRL,
3498 RTW89_PCIE_BIT_L1SUB);
3499 if (ret)
3500 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
3501 enable ? "set" : "unset", ret);
3502 } else if (chip_id == RTL8852C) {
3503 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
3504 RTW89_PCIE_BIT_ASPM_L11 |
3505 RTW89_PCIE_BIT_PCI_L11);
3506 if (ret)
3507 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
3508 if (enable)
3509 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3510 B_AX_L1SUB_DISABLE);
3511 else
3512 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3513 B_AX_L1SUB_DISABLE);
3514 }
3515 }
3516
rtw89_pci_l1ss_cfg(struct rtw89_dev * rtwdev)3517 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
3518 {
3519 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3520 struct pci_dev *pdev = rtwpci->pdev;
3521 u32 l1ss_cap_ptr, l1ss_ctrl;
3522
3523 if (rtw89_pci_disable_l1ss)
3524 return;
3525
3526 l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
3527 if (!l1ss_cap_ptr)
3528 return;
3529
3530 pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
3531
3532 if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
3533 rtw89_pci_l1ss_set(rtwdev, true);
3534 }
3535
rtw89_pci_poll_io_idle(struct rtw89_dev * rtwdev)3536 static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
3537 {
3538 int ret = 0;
3539 u32 sts;
3540 u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
3541
3542 ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
3543 10, 1000, false, rtwdev,
3544 R_AX_PCIE_DMA_BUSY1);
3545 if (ret) {
3546 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
3547 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
3548 return -EINVAL;
3549 }
3550 return ret;
3551 }
3552
rtw89_pci_lv1rst_stop_dma(struct rtw89_dev * rtwdev)3553 static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
3554 {
3555 u32 val;
3556 int ret;
3557
3558 if (rtwdev->chip->chip_id == RTL8852C)
3559 return 0;
3560
3561 rtw89_pci_ctrl_dma_all(rtwdev, false);
3562 ret = rtw89_pci_poll_io_idle(rtwdev);
3563 if (ret) {
3564 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3565 rtw89_debug(rtwdev, RTW89_DBG_HCI,
3566 "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
3567 R_AX_DBG_ERR_FLAG, val);
3568 if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
3569 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
3570 if (val & B_AX_RX_STUCK)
3571 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
3572 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3573 ret = rtw89_pci_poll_io_idle(rtwdev);
3574 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3575 rtw89_debug(rtwdev, RTW89_DBG_HCI,
3576 "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
3577 R_AX_DBG_ERR_FLAG, val);
3578 }
3579
3580 return ret;
3581 }
3582
3583
3584
rtw89_pci_rst_bdram(struct rtw89_dev * rtwdev)3585 static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)
3586 {
3587 int ret = 0;
3588 u32 val32, sts;
3589
3590 val32 = B_AX_RST_BDRAM;
3591 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
3592
3593 ret = read_poll_timeout_atomic(rtw89_read32, sts,
3594 (sts & B_AX_RST_BDRAM) == 0x0, 1, 100,
3595 true, rtwdev, R_AX_PCIE_INIT_CFG1);
3596 return ret;
3597 }
3598
rtw89_pci_lv1rst_start_dma(struct rtw89_dev * rtwdev)3599 static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
3600 {
3601 u32 ret;
3602
3603 if (rtwdev->chip->chip_id == RTL8852C)
3604 return 0;
3605
3606 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
3607 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3608 rtw89_pci_clr_idx_all(rtwdev);
3609
3610 ret = rtw89_pci_rst_bdram(rtwdev);
3611 if (ret)
3612 return ret;
3613
3614 rtw89_pci_ctrl_dma_all(rtwdev, true);
3615 return ret;
3616 }
3617
rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev * rtwdev,enum rtw89_lv1_rcvy_step step)3618 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
3619 enum rtw89_lv1_rcvy_step step)
3620 {
3621 int ret;
3622
3623 switch (step) {
3624 case RTW89_LV1_RCVY_STEP_1:
3625 ret = rtw89_pci_lv1rst_stop_dma(rtwdev);
3626 if (ret)
3627 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
3628
3629 break;
3630
3631 case RTW89_LV1_RCVY_STEP_2:
3632 ret = rtw89_pci_lv1rst_start_dma(rtwdev);
3633 if (ret)
3634 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
3635 break;
3636
3637 default:
3638 return -EINVAL;
3639 }
3640
3641 return ret;
3642 }
3643
rtw89_pci_ops_dump_err_status(struct rtw89_dev * rtwdev)3644 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
3645 {
3646 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
3647 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
3648 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
3649 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
3650 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
3651 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
3652 }
3653
rtw89_pci_napi_poll(struct napi_struct * napi,int budget)3654 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
3655 {
3656 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
3657 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3658 unsigned long flags;
3659 int work_done;
3660
3661 rtwdev->napi_budget_countdown = budget;
3662
3663 rtw89_pci_clear_isr0(rtwdev, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT);
3664 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3665 if (work_done == budget)
3666 return budget;
3667
3668 rtw89_pci_clear_isr0(rtwdev, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | B_AX_RDU_INT);
3669 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3670 if (work_done < budget && napi_complete_done(napi, work_done)) {
3671 spin_lock_irqsave(&rtwpci->irq_lock, flags);
3672 if (likely(rtwpci->running))
3673 rtw89_chip_enable_intr(rtwdev, rtwpci);
3674 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
3675 }
3676
3677 return work_done;
3678 }
3679
rtw89_pci_suspend(struct device * dev)3680 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
3681 {
3682 struct ieee80211_hw *hw = dev_get_drvdata(dev);
3683 struct rtw89_dev *rtwdev = hw->priv;
3684 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3685
3686 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3687 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3688 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3689 if (chip_id == RTL8852A || chip_id == RTL8852B) {
3690 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
3691 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3692 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
3693 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3694 } else {
3695 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3696 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3697 }
3698
3699 return 0;
3700 }
3701
rtw89_pci_l2_hci_ldo(struct rtw89_dev * rtwdev)3702 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
3703 {
3704 if (rtwdev->chip->chip_id == RTL8852C)
3705 return;
3706
3707 /* Hardware need write the reg twice to ensure the setting work */
3708 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3709 RTW89_PCIE_BIT_CFG_RST_MSTATE);
3710 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3711 RTW89_PCIE_BIT_CFG_RST_MSTATE);
3712 }
3713
rtw89_pci_resume(struct device * dev)3714 static int __maybe_unused rtw89_pci_resume(struct device *dev)
3715 {
3716 struct ieee80211_hw *hw = dev_get_drvdata(dev);
3717 struct rtw89_dev *rtwdev = hw->priv;
3718 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3719
3720 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3721 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3722 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3723 if (chip_id == RTL8852A || chip_id == RTL8852B) {
3724 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
3725 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3726 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
3727 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3728 } else {
3729 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3730 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3731 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3732 B_AX_SEL_REQ_ENTR_L1);
3733 }
3734 rtw89_pci_l2_hci_ldo(rtwdev);
3735 rtw89_pci_filter_out(rtwdev);
3736 rtw89_pci_link_cfg(rtwdev);
3737 rtw89_pci_l1ss_cfg(rtwdev);
3738
3739 return 0;
3740 }
3741
3742 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
3743 EXPORT_SYMBOL(rtw89_pm_ops);
3744
3745 static const struct rtw89_hci_ops rtw89_pci_ops = {
3746 .tx_write = rtw89_pci_ops_tx_write,
3747 .tx_kick_off = rtw89_pci_ops_tx_kick_off,
3748 .flush_queues = rtw89_pci_ops_flush_queues,
3749 .reset = rtw89_pci_ops_reset,
3750 .start = rtw89_pci_ops_start,
3751 .stop = rtw89_pci_ops_stop,
3752 .pause = rtw89_pci_ops_pause,
3753 .switch_mode = rtw89_pci_ops_switch_mode,
3754 .recalc_int_mit = rtw89_pci_recalc_int_mit,
3755
3756 .read8 = rtw89_pci_ops_read8,
3757 .read16 = rtw89_pci_ops_read16,
3758 .read32 = rtw89_pci_ops_read32,
3759 .write8 = rtw89_pci_ops_write8,
3760 .write16 = rtw89_pci_ops_write16,
3761 .write32 = rtw89_pci_ops_write32,
3762
3763 .mac_pre_init = rtw89_pci_ops_mac_pre_init,
3764 .mac_post_init = rtw89_pci_ops_mac_post_init,
3765 .deinit = rtw89_pci_ops_deinit,
3766
3767 .check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
3768 .mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery,
3769 .dump_err_status = rtw89_pci_ops_dump_err_status,
3770 .napi_poll = rtw89_pci_napi_poll,
3771
3772 .recovery_start = rtw89_pci_ops_recovery_start,
3773 .recovery_complete = rtw89_pci_ops_recovery_complete,
3774 };
3775
rtw89_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)3776 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3777 {
3778 struct rtw89_dev *rtwdev;
3779 const struct rtw89_driver_info *info;
3780 const struct rtw89_pci_info *pci_info;
3781 int ret;
3782
3783 info = (const struct rtw89_driver_info *)id->driver_data;
3784
3785 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
3786 sizeof(struct rtw89_pci),
3787 info->chip);
3788 if (!rtwdev) {
3789 dev_err(&pdev->dev, "failed to allocate hw\n");
3790 return -ENOMEM;
3791 }
3792
3793 pci_info = info->bus.pci;
3794
3795 rtwdev->pci_info = info->bus.pci;
3796 rtwdev->hci.ops = &rtw89_pci_ops;
3797 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
3798 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
3799 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
3800
3801 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
3802
3803 ret = rtw89_core_init(rtwdev);
3804 if (ret) {
3805 rtw89_err(rtwdev, "failed to initialise core\n");
3806 goto err_release_hw;
3807 }
3808
3809 ret = rtw89_pci_claim_device(rtwdev, pdev);
3810 if (ret) {
3811 rtw89_err(rtwdev, "failed to claim pci device\n");
3812 goto err_core_deinit;
3813 }
3814
3815 ret = rtw89_pci_setup_resource(rtwdev, pdev);
3816 if (ret) {
3817 rtw89_err(rtwdev, "failed to setup pci resource\n");
3818 goto err_declaim_pci;
3819 }
3820
3821 ret = rtw89_chip_info_setup(rtwdev);
3822 if (ret) {
3823 rtw89_err(rtwdev, "failed to setup chip information\n");
3824 goto err_clear_resource;
3825 }
3826
3827 rtw89_pci_filter_out(rtwdev);
3828 rtw89_pci_link_cfg(rtwdev);
3829 rtw89_pci_l1ss_cfg(rtwdev);
3830
3831 ret = rtw89_core_register(rtwdev);
3832 if (ret) {
3833 rtw89_err(rtwdev, "failed to register core\n");
3834 goto err_clear_resource;
3835 }
3836
3837 rtw89_core_napi_init(rtwdev);
3838
3839 ret = rtw89_pci_request_irq(rtwdev, pdev);
3840 if (ret) {
3841 rtw89_err(rtwdev, "failed to request pci irq\n");
3842 goto err_unregister;
3843 }
3844
3845 return 0;
3846
3847 err_unregister:
3848 rtw89_core_napi_deinit(rtwdev);
3849 rtw89_core_unregister(rtwdev);
3850 err_clear_resource:
3851 rtw89_pci_clear_resource(rtwdev, pdev);
3852 err_declaim_pci:
3853 rtw89_pci_declaim_device(rtwdev, pdev);
3854 err_core_deinit:
3855 rtw89_core_deinit(rtwdev);
3856 err_release_hw:
3857 rtw89_free_ieee80211_hw(rtwdev);
3858
3859 return ret;
3860 }
3861 EXPORT_SYMBOL(rtw89_pci_probe);
3862
rtw89_pci_remove(struct pci_dev * pdev)3863 void rtw89_pci_remove(struct pci_dev *pdev)
3864 {
3865 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
3866 struct rtw89_dev *rtwdev;
3867
3868 rtwdev = hw->priv;
3869
3870 rtw89_pci_free_irq(rtwdev, pdev);
3871 rtw89_core_napi_deinit(rtwdev);
3872 rtw89_core_unregister(rtwdev);
3873 rtw89_pci_clear_resource(rtwdev, pdev);
3874 rtw89_pci_declaim_device(rtwdev, pdev);
3875 rtw89_core_deinit(rtwdev);
3876 rtw89_free_ieee80211_hw(rtwdev);
3877 }
3878 EXPORT_SYMBOL(rtw89_pci_remove);
3879
3880 MODULE_AUTHOR("Realtek Corporation");
3881 MODULE_DESCRIPTION("Realtek 802.11ax wireless PCI driver");
3882 MODULE_LICENSE("Dual BSD/GPL");
3883
