Lines Matching +full:10 +full:gbase +full:- +full:kr
1 // SPDX-License-Identifier: GPL-2.0
76 writel(data, priv->swth_base[0] + offset); in mvpp2_write()
81 return readl(priv->swth_base[0] + offset); in mvpp2_read()
86 return readl_relaxed(priv->swth_base[0] + offset); in mvpp2_read_relaxed()
91 return cpu % priv->nthreads; in mvpp2_cpu_to_thread()
96 writel(data, priv->cm3_base + offset); in mvpp2_cm3_write()
101 return readl(priv->cm3_base + offset); in mvpp2_cm3_read()
124 * - per-thread registers, where each thread has its own copy of the
140 * - global registers that must be accessed through a specific thread
141 * window, because they are related to an access to a per-thread
161 writel(data, priv->swth_base[thread] + offset); in mvpp2_thread_write()
167 return readl(priv->swth_base[thread] + offset); in mvpp2_thread_read()
173 writel_relaxed(data, priv->swth_base[thread] + offset); in mvpp2_thread_write_relaxed()
179 return readl_relaxed(priv->swth_base[thread] + offset); in mvpp2_thread_read_relaxed()
185 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_dma_addr_get()
186 return le32_to_cpu(tx_desc->pp21.buf_dma_addr); in mvpp2_txdesc_dma_addr_get()
188 return le64_to_cpu(tx_desc->pp22.buf_dma_addr_ptp) & in mvpp2_txdesc_dma_addr_get()
201 if (port->priv->hw_version == MVPP21) { in mvpp2_txdesc_dma_addr_set()
202 tx_desc->pp21.buf_dma_addr = cpu_to_le32(addr); in mvpp2_txdesc_dma_addr_set()
203 tx_desc->pp21.packet_offset = offset; in mvpp2_txdesc_dma_addr_set()
207 tx_desc->pp22.buf_dma_addr_ptp &= ~cpu_to_le64(MVPP2_DESC_DMA_MASK); in mvpp2_txdesc_dma_addr_set()
208 tx_desc->pp22.buf_dma_addr_ptp |= val; in mvpp2_txdesc_dma_addr_set()
209 tx_desc->pp22.packet_offset = offset; in mvpp2_txdesc_dma_addr_set()
216 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_size_get()
217 return le16_to_cpu(tx_desc->pp21.data_size); in mvpp2_txdesc_size_get()
219 return le16_to_cpu(tx_desc->pp22.data_size); in mvpp2_txdesc_size_get()
226 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_size_set()
227 tx_desc->pp21.data_size = cpu_to_le16(size); in mvpp2_txdesc_size_set()
229 tx_desc->pp22.data_size = cpu_to_le16(size); in mvpp2_txdesc_size_set()
236 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_txq_set()
237 tx_desc->pp21.phys_txq = txq; in mvpp2_txdesc_txq_set()
239 tx_desc->pp22.phys_txq = txq; in mvpp2_txdesc_txq_set()
246 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_cmd_set()
247 tx_desc->pp21.command = cpu_to_le32(command); in mvpp2_txdesc_cmd_set()
249 tx_desc->pp22.command = cpu_to_le32(command); in mvpp2_txdesc_cmd_set()
255 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_offset_get()
256 return tx_desc->pp21.packet_offset; in mvpp2_txdesc_offset_get()
258 return tx_desc->pp22.packet_offset; in mvpp2_txdesc_offset_get()
264 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_dma_addr_get()
265 return le32_to_cpu(rx_desc->pp21.buf_dma_addr); in mvpp2_rxdesc_dma_addr_get()
267 return le64_to_cpu(rx_desc->pp22.buf_dma_addr_key_hash) & in mvpp2_rxdesc_dma_addr_get()
274 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_cookie_get()
275 return le32_to_cpu(rx_desc->pp21.buf_cookie); in mvpp2_rxdesc_cookie_get()
277 return le64_to_cpu(rx_desc->pp22.buf_cookie_misc) & in mvpp2_rxdesc_cookie_get()
284 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_size_get()
285 return le16_to_cpu(rx_desc->pp21.data_size); in mvpp2_rxdesc_size_get()
287 return le16_to_cpu(rx_desc->pp22.data_size); in mvpp2_rxdesc_size_get()
293 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_status_get()
294 return le32_to_cpu(rx_desc->pp21.status); in mvpp2_rxdesc_status_get()
296 return le32_to_cpu(rx_desc->pp22.status); in mvpp2_rxdesc_status_get()
301 txq_pcpu->txq_get_index++; in mvpp2_txq_inc_get()
302 if (txq_pcpu->txq_get_index == txq_pcpu->size) in mvpp2_txq_inc_get()
303 txq_pcpu->txq_get_index = 0; in mvpp2_txq_inc_get()
313 txq_pcpu->buffs + txq_pcpu->txq_put_index; in mvpp2_txq_inc_put()
314 tx_buf->type = buf_type; in mvpp2_txq_inc_put()
316 tx_buf->skb = data; in mvpp2_txq_inc_put()
318 tx_buf->xdpf = data; in mvpp2_txq_inc_put()
319 tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc); in mvpp2_txq_inc_put()
320 tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) + in mvpp2_txq_inc_put()
322 txq_pcpu->txq_put_index++; in mvpp2_txq_inc_put()
323 if (txq_pcpu->txq_put_index == txq_pcpu->size) in mvpp2_txq_inc_put()
324 txq_pcpu->txq_put_index = 0; in mvpp2_txq_inc_put()
332 if (priv->hw_version >= MVPP22 && queue_mode == MVPP2_QDIST_SINGLE_MODE) in mvpp2_get_nrxqs()
350 return MVPP2_MAX_TCONT + port->id; in mvpp2_egress_port()
366 if (likely(pool->frag_size <= PAGE_SIZE)) in mvpp2_frag_alloc()
367 return netdev_alloc_frag(pool->frag_size); in mvpp2_frag_alloc()
369 return kmalloc(pool->frag_size, GFP_ATOMIC); in mvpp2_frag_alloc()
377 else if (likely(pool->frag_size <= PAGE_SIZE)) in mvpp2_frag_free()
395 return -EINVAL; in mvpp2_bm_pool_create()
400 if (priv->hw_version == MVPP21) in mvpp2_bm_pool_create()
401 bm_pool->size_bytes = 2 * sizeof(u32) * size; in mvpp2_bm_pool_create()
403 bm_pool->size_bytes = 2 * sizeof(u64) * size; in mvpp2_bm_pool_create()
405 bm_pool->virt_addr = dma_alloc_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_create()
406 &bm_pool->dma_addr, in mvpp2_bm_pool_create()
408 if (!bm_pool->virt_addr) in mvpp2_bm_pool_create()
409 return -ENOMEM; in mvpp2_bm_pool_create()
411 if (!IS_ALIGNED((unsigned long)bm_pool->virt_addr, in mvpp2_bm_pool_create()
413 dma_free_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_create()
414 bm_pool->virt_addr, bm_pool->dma_addr); in mvpp2_bm_pool_create()
416 bm_pool->id, MVPP2_BM_POOL_PTR_ALIGN); in mvpp2_bm_pool_create()
417 return -ENOMEM; in mvpp2_bm_pool_create()
420 mvpp2_write(priv, MVPP2_BM_POOL_BASE_REG(bm_pool->id), in mvpp2_bm_pool_create()
421 lower_32_bits(bm_pool->dma_addr)); in mvpp2_bm_pool_create()
422 mvpp2_write(priv, MVPP2_BM_POOL_SIZE_REG(bm_pool->id), size); in mvpp2_bm_pool_create()
424 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id)); in mvpp2_bm_pool_create()
431 if (priv->hw_version == MVPP23) { in mvpp2_bm_pool_create()
439 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val); in mvpp2_bm_pool_create()
441 bm_pool->size = size; in mvpp2_bm_pool_create()
442 bm_pool->pkt_size = 0; in mvpp2_bm_pool_create()
443 bm_pool->buf_num = 0; in mvpp2_bm_pool_create()
455 bm_pool->buf_size = buf_size; in mvpp2_bm_pool_bufsize_set()
458 mvpp2_write(priv, MVPP2_POOL_BUF_SIZE_REG(bm_pool->id), val); in mvpp2_bm_pool_bufsize_set()
469 MVPP2_BM_PHY_ALLOC_REG(bm_pool->id)); in mvpp2_bm_bufs_get_addrs()
472 if (priv->hw_version >= MVPP22) { in mvpp2_bm_bufs_get_addrs()
498 if (buf_num > bm_pool->buf_num) { in mvpp2_bm_bufs_free()
500 bm_pool->id, buf_num); in mvpp2_bm_bufs_free()
501 buf_num = bm_pool->buf_num; in mvpp2_bm_bufs_free()
504 if (priv->percpu_pools) in mvpp2_bm_bufs_free()
505 pp = priv->page_pool[bm_pool->id]; in mvpp2_bm_bufs_free()
517 bm_pool->buf_size, DMA_FROM_DEVICE); in mvpp2_bm_bufs_free()
527 bm_pool->buf_num -= i; in mvpp2_bm_bufs_free()
535 buf_num += mvpp2_read(priv, MVPP2_BM_POOL_PTRS_NUM_REG(bm_pool->id)) & in mvpp2_check_hw_buf_num()
537 buf_num += mvpp2_read(priv, MVPP2_BM_BPPI_PTRS_NUM_REG(bm_pool->id)) & in mvpp2_check_hw_buf_num()
561 bm_pool->id, bm_pool->buf_num); in mvpp2_bm_pool_destroy()
565 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id)); in mvpp2_bm_pool_destroy()
567 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val); in mvpp2_bm_pool_destroy()
569 if (priv->percpu_pools) { in mvpp2_bm_pool_destroy()
570 page_pool_destroy(priv->page_pool[bm_pool->id]); in mvpp2_bm_pool_destroy()
571 priv->page_pool[bm_pool->id] = NULL; in mvpp2_bm_pool_destroy()
574 dma_free_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_destroy()
575 bm_pool->virt_addr, in mvpp2_bm_pool_destroy()
576 bm_pool->dma_addr); in mvpp2_bm_pool_destroy()
585 if (priv->percpu_pools) in mvpp2_bm_pools_init()
591 bm_pool = &priv->bm_pools[i]; in mvpp2_bm_pools_init()
592 bm_pool->id = i; in mvpp2_bm_pools_init()
602 for (i = i - 1; i >= 0; i--) in mvpp2_bm_pools_init()
603 mvpp2_bm_pool_destroy(dev, priv, &priv->bm_pools[i]); in mvpp2_bm_pools_init()
623 if (priv->percpu_pools) { in mvpp2_bm_init()
624 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_init()
625 port = priv->port_list[i]; in mvpp2_bm_init()
626 if (port->xdp_prog) { in mvpp2_bm_init()
637 priv->page_pool[i] = in mvpp2_bm_init()
642 if (IS_ERR(priv->page_pool[i])) { in mvpp2_bm_init()
646 page_pool_destroy(priv->page_pool[j]); in mvpp2_bm_init()
647 priv->page_pool[j] = NULL; in mvpp2_bm_init()
649 return PTR_ERR(priv->page_pool[i]); in mvpp2_bm_init()
655 priv->percpu_pools ? "per-cpu" : "shared"); in mvpp2_bm_init()
665 priv->bm_pools = devm_kcalloc(dev, poolnum, in mvpp2_bm_init()
666 sizeof(*priv->bm_pools), GFP_KERNEL); in mvpp2_bm_init()
667 if (!priv->bm_pools) in mvpp2_bm_init()
668 return -ENOMEM; in mvpp2_bm_init()
670 if (priv->hw_version == MVPP23) in mvpp2_bm_init()
702 prxq = port->rxqs[lrxq]->id; in mvpp2_rxq_long_pool_set()
704 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_long_pool_set()
709 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_long_pool_set()
712 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_long_pool_set()
723 prxq = port->rxqs[lrxq]->id; in mvpp2_rxq_short_pool_set()
725 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_short_pool_set()
730 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_short_pool_set()
733 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_short_pool_set()
756 dma_addr = dma_map_single(port->dev->dev.parent, data, in mvpp2_buf_alloc()
757 MVPP2_RX_BUF_SIZE(bm_pool->pkt_size), in mvpp2_buf_alloc()
759 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) { in mvpp2_buf_alloc()
774 int fq = port->first_rxq; in mvpp2_rxq_enable_fc()
777 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_rxq_enable_fc()
780 * If Flow control was enabled, it would be re-enabled. in mvpp2_rxq_enable_fc()
782 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_enable_fc()
785 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_enable_fc()
788 for (q = 0; q < port->nrxqs; q++) { in mvpp2_rxq_enable_fc()
792 mvpp2_cm3_write(port->priv, MSS_RXQ_TRESH_REG(q, fq), val); in mvpp2_rxq_enable_fc()
794 val = mvpp2_cm3_read(port->priv, MSS_RXQ_ASS_REG(q, fq)); in mvpp2_rxq_enable_fc()
797 val |= (port->id << MSS_RXQ_ASS_Q_BASE(q, fq)); in mvpp2_rxq_enable_fc()
809 host_id = port->nqvecs; in mvpp2_rxq_enable_fc()
819 mvpp2_cm3_write(port->priv, MSS_RXQ_ASS_REG(q, fq), val); in mvpp2_rxq_enable_fc()
823 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_enable_fc()
826 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_enable_fc()
828 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_rxq_enable_fc()
836 int fq = port->first_rxq; in mvpp2_rxq_disable_fc()
838 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_rxq_disable_fc()
841 * If Flow control was enabled, it would be re-enabled. in mvpp2_rxq_disable_fc()
843 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_disable_fc()
846 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_disable_fc()
849 for (q = 0; q < port->nrxqs; q++) { in mvpp2_rxq_disable_fc()
853 mvpp2_cm3_write(port->priv, MSS_RXQ_TRESH_REG(q, fq), val); in mvpp2_rxq_disable_fc()
855 val = mvpp2_cm3_read(port->priv, MSS_RXQ_ASS_REG(q, fq)); in mvpp2_rxq_disable_fc()
862 mvpp2_cm3_write(port->priv, MSS_RXQ_ASS_REG(q, fq), val); in mvpp2_rxq_disable_fc()
866 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_disable_fc()
869 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_disable_fc()
871 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_rxq_disable_fc()
882 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_bm_pool_update_fc()
885 * If Flow control were enabled, it would be re-enabled. in mvpp2_bm_pool_update_fc()
887 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_bm_pool_update_fc()
890 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_bm_pool_update_fc()
895 val = mvpp2_cm3_read(port->priv, MSS_BUF_POOL_REG(pool->id)); in mvpp2_bm_pool_update_fc()
896 val |= MSS_BUF_POOL_PORT_OFFS(port->id); in mvpp2_bm_pool_update_fc()
901 mvpp2_cm3_write(port->priv, MSS_BUF_POOL_REG(pool->id), val); in mvpp2_bm_pool_update_fc()
904 val = mvpp2_cm3_read(port->priv, MSS_BUF_POOL_REG(pool->id)); in mvpp2_bm_pool_update_fc()
905 val &= ~MSS_BUF_POOL_PORT_OFFS(port->id); in mvpp2_bm_pool_update_fc()
910 if (!pool->buf_num) { in mvpp2_bm_pool_update_fc()
915 mvpp2_cm3_write(port->priv, MSS_BUF_POOL_REG(pool->id), val); in mvpp2_bm_pool_update_fc()
919 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_bm_pool_update_fc()
922 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_bm_pool_update_fc()
924 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_bm_pool_update_fc()
933 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_pool_update_priv_fc()
934 port = priv->port_list[i]; in mvpp2_bm_pool_update_priv_fc()
935 if (port->priv->percpu_pools) { in mvpp2_bm_pool_update_priv_fc()
936 for (i = 0; i < port->nrxqs; i++) in mvpp2_bm_pool_update_priv_fc()
937 mvpp2_bm_pool_update_fc(port, &port->priv->bm_pools[i], in mvpp2_bm_pool_update_priv_fc()
938 port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
940 mvpp2_bm_pool_update_fc(port, port->pool_long, port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
941 mvpp2_bm_pool_update_fc(port, port->pool_short, port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
966 usleep_range(10, 20); in mvpp2_enable_global_fc()
970 priv->global_tx_fc = false; in mvpp2_enable_global_fc()
971 return -EOPNOTSUPP; in mvpp2_enable_global_fc()
979 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_bm_pool_put()
982 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_bm_pool_put()
983 spin_lock_irqsave(&port->bm_lock[thread], flags); in mvpp2_bm_pool_put()
985 if (port->priv->hw_version >= MVPP22) { in mvpp2_bm_pool_put()
997 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1006 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1008 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1011 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_bm_pool_put()
1012 spin_unlock_irqrestore(&port->bm_lock[thread], flags); in mvpp2_bm_pool_put()
1027 if (port->priv->percpu_pools && in mvpp2_bm_bufs_add()
1028 bm_pool->pkt_size > MVPP2_BM_LONG_PKT_SIZE) { in mvpp2_bm_bufs_add()
1029 netdev_err(port->dev, in mvpp2_bm_bufs_add()
1030 "attempted to use jumbo frames with per-cpu pools"); in mvpp2_bm_bufs_add()
1034 buf_size = MVPP2_RX_BUF_SIZE(bm_pool->pkt_size); in mvpp2_bm_bufs_add()
1038 (buf_num + bm_pool->buf_num > bm_pool->size)) { in mvpp2_bm_bufs_add()
1039 netdev_err(port->dev, in mvpp2_bm_bufs_add()
1041 buf_num, bm_pool->id); in mvpp2_bm_bufs_add()
1045 if (port->priv->percpu_pools) in mvpp2_bm_bufs_add()
1046 pp = port->priv->page_pool[bm_pool->id]; in mvpp2_bm_bufs_add()
1053 mvpp2_bm_pool_put(port, bm_pool->id, dma_addr, in mvpp2_bm_bufs_add()
1058 bm_pool->buf_num += i; in mvpp2_bm_bufs_add()
1060 netdev_dbg(port->dev, in mvpp2_bm_bufs_add()
1062 bm_pool->id, bm_pool->pkt_size, buf_size, total_size); in mvpp2_bm_bufs_add()
1064 netdev_dbg(port->dev, in mvpp2_bm_bufs_add()
1066 bm_pool->id, i, buf_num); in mvpp2_bm_bufs_add()
1076 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool]; in mvpp2_bm_pool_use()
1079 if ((port->priv->percpu_pools && pool > mvpp2_get_nrxqs(port->priv) * 2) || in mvpp2_bm_pool_use()
1080 (!port->priv->percpu_pools && pool >= MVPP2_BM_POOLS_NUM)) { in mvpp2_bm_pool_use()
1081 netdev_err(port->dev, "Invalid pool %d\n", pool); in mvpp2_bm_pool_use()
1088 if (new_pool->pkt_size == 0) { in mvpp2_bm_pool_use()
1094 pkts_num = new_pool->buf_num; in mvpp2_bm_pool_use()
1096 if (port->priv->percpu_pools) { in mvpp2_bm_pool_use()
1097 if (pool < port->nrxqs) in mvpp2_bm_pool_use()
1105 mvpp2_bm_bufs_free(port->dev->dev.parent, in mvpp2_bm_pool_use()
1106 port->priv, new_pool, pkts_num); in mvpp2_bm_pool_use()
1109 new_pool->pkt_size = pkt_size; in mvpp2_bm_pool_use()
1110 new_pool->frag_size = in mvpp2_bm_pool_use()
1118 new_pool->id, num, pkts_num); in mvpp2_bm_pool_use()
1123 mvpp2_bm_pool_bufsize_set(port->priv, new_pool, in mvpp2_bm_pool_use()
1124 MVPP2_RX_BUF_SIZE(new_pool->pkt_size)); in mvpp2_bm_pool_use()
1133 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool]; in mvpp2_bm_pool_use_percpu()
1136 if (pool > port->nrxqs * 2) { in mvpp2_bm_pool_use_percpu()
1137 netdev_err(port->dev, "Invalid pool %d\n", pool); in mvpp2_bm_pool_use_percpu()
1144 if (new_pool->pkt_size == 0) { in mvpp2_bm_pool_use_percpu()
1150 pkts_num = new_pool->buf_num; in mvpp2_bm_pool_use_percpu()
1154 mvpp2_bm_bufs_free(port->dev->dev.parent, in mvpp2_bm_pool_use_percpu()
1155 port->priv, new_pool, pkts_num); in mvpp2_bm_pool_use_percpu()
1157 new_pool->pkt_size = pkt_size; in mvpp2_bm_pool_use_percpu()
1158 new_pool->frag_size = in mvpp2_bm_pool_use_percpu()
1166 new_pool->id, num, pkts_num); in mvpp2_bm_pool_use_percpu()
1171 mvpp2_bm_pool_bufsize_set(port->priv, new_pool, in mvpp2_bm_pool_use_percpu()
1172 MVPP2_RX_BUF_SIZE(new_pool->pkt_size)); in mvpp2_bm_pool_use_percpu()
1184 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool in mvpp2_swf_bm_pool_init_shared()
1185 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool in mvpp2_swf_bm_pool_init_shared()
1187 if (port->pkt_size > MVPP2_BM_LONG_PKT_SIZE) { in mvpp2_swf_bm_pool_init_shared()
1195 if (!port->pool_long) { in mvpp2_swf_bm_pool_init_shared()
1196 port->pool_long = in mvpp2_swf_bm_pool_init_shared()
1199 if (!port->pool_long) in mvpp2_swf_bm_pool_init_shared()
1200 return -ENOMEM; in mvpp2_swf_bm_pool_init_shared()
1202 port->pool_long->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_shared()
1204 for (rxq = 0; rxq < port->nrxqs; rxq++) in mvpp2_swf_bm_pool_init_shared()
1205 mvpp2_rxq_long_pool_set(port, rxq, port->pool_long->id); in mvpp2_swf_bm_pool_init_shared()
1208 if (!port->pool_short) { in mvpp2_swf_bm_pool_init_shared()
1209 port->pool_short = in mvpp2_swf_bm_pool_init_shared()
1212 if (!port->pool_short) in mvpp2_swf_bm_pool_init_shared()
1213 return -ENOMEM; in mvpp2_swf_bm_pool_init_shared()
1215 port->pool_short->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_shared()
1217 for (rxq = 0; rxq < port->nrxqs; rxq++) in mvpp2_swf_bm_pool_init_shared()
1219 port->pool_short->id); in mvpp2_swf_bm_pool_init_shared()
1231 for (i = 0; i < port->nrxqs; i++) { in mvpp2_swf_bm_pool_init_percpu()
1235 return -ENOMEM; in mvpp2_swf_bm_pool_init_percpu()
1237 bm_pool->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_percpu()
1238 mvpp2_rxq_short_pool_set(port, i, bm_pool->id); in mvpp2_swf_bm_pool_init_percpu()
1241 for (i = 0; i < port->nrxqs; i++) { in mvpp2_swf_bm_pool_init_percpu()
1242 bm_pool = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_LONG, i + port->nrxqs, in mvpp2_swf_bm_pool_init_percpu()
1245 return -ENOMEM; in mvpp2_swf_bm_pool_init_percpu()
1247 bm_pool->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_percpu()
1248 mvpp2_rxq_long_pool_set(port, i, bm_pool->id); in mvpp2_swf_bm_pool_init_percpu()
1251 port->pool_long = NULL; in mvpp2_swf_bm_pool_init_percpu()
1252 port->pool_short = NULL; in mvpp2_swf_bm_pool_init_percpu()
1259 if (port->priv->percpu_pools) in mvpp2_swf_bm_pool_init()
1276 if (new_long_pool == MVPP2_BM_JUMBO && port->id != 0) { in mvpp2_set_hw_csum()
1277 port->dev->features &= ~csums; in mvpp2_set_hw_csum()
1278 port->dev->hw_features &= ~csums; in mvpp2_set_hw_csum()
1280 port->dev->features |= csums; in mvpp2_set_hw_csum()
1281 port->dev->hw_features |= csums; in mvpp2_set_hw_csum()
1291 if (port->priv->percpu_pools) in mvpp2_bm_update_mtu()
1295 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool in mvpp2_bm_update_mtu()
1296 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool in mvpp2_bm_update_mtu()
1303 if (new_long_pool != port->pool_long->id) { in mvpp2_bm_update_mtu()
1304 if (port->tx_fc) { in mvpp2_bm_update_mtu()
1307 port->pool_short, in mvpp2_bm_update_mtu()
1310 mvpp2_bm_pool_update_fc(port, port->pool_long, in mvpp2_bm_update_mtu()
1315 port->pool_long = mvpp2_bm_pool_use(port, port->pool_long->id, in mvpp2_bm_update_mtu()
1316 port->pool_long->pkt_size); in mvpp2_bm_update_mtu()
1317 port->pool_long->port_map &= ~BIT(port->id); in mvpp2_bm_update_mtu()
1318 port->pool_long = NULL; in mvpp2_bm_update_mtu()
1320 port->pool_short = mvpp2_bm_pool_use(port, port->pool_short->id, in mvpp2_bm_update_mtu()
1321 port->pool_short->pkt_size); in mvpp2_bm_update_mtu()
1322 port->pool_short->port_map &= ~BIT(port->id); in mvpp2_bm_update_mtu()
1323 port->pool_short = NULL; in mvpp2_bm_update_mtu()
1325 port->pkt_size = pkt_size; in mvpp2_bm_update_mtu()
1332 if (port->tx_fc) { in mvpp2_bm_update_mtu()
1334 mvpp2_bm_pool_update_fc(port, port->pool_long, in mvpp2_bm_update_mtu()
1337 mvpp2_bm_pool_update_fc(port, port->pool_short, in mvpp2_bm_update_mtu()
1342 if (new_long_pool == MVPP2_BM_JUMBO && port->id != 0) { in mvpp2_bm_update_mtu()
1343 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); in mvpp2_bm_update_mtu()
1344 dev->hw_features &= ~(NETIF_F_IP_CSUM | in mvpp2_bm_update_mtu()
1347 dev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; in mvpp2_bm_update_mtu()
1348 dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; in mvpp2_bm_update_mtu()
1353 dev->mtu = mtu; in mvpp2_bm_update_mtu()
1354 dev->wanted_features = dev->features; in mvpp2_bm_update_mtu()
1364 for (i = 0; i < port->nqvecs; i++) in mvpp2_interrupts_enable()
1365 sw_thread_mask |= port->qvecs[i].sw_thread_mask; in mvpp2_interrupts_enable()
1367 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_interrupts_enable()
1375 for (i = 0; i < port->nqvecs; i++) in mvpp2_interrupts_disable()
1376 sw_thread_mask |= port->qvecs[i].sw_thread_mask; in mvpp2_interrupts_disable()
1378 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_interrupts_disable()
1384 struct mvpp2_port *port = qvec->port; in mvpp2_qvec_interrupt_enable()
1386 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_qvec_interrupt_enable()
1387 MVPP2_ISR_ENABLE_INTERRUPT(qvec->sw_thread_mask)); in mvpp2_qvec_interrupt_enable()
1392 struct mvpp2_port *port = qvec->port; in mvpp2_qvec_interrupt_disable()
1394 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_qvec_interrupt_disable()
1395 MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask)); in mvpp2_qvec_interrupt_disable()
1409 if (cpu > port->priv->nthreads) in mvpp2_interrupts_mask()
1412 thread = mvpp2_cpu_to_thread(port->priv, cpu); in mvpp2_interrupts_mask()
1414 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_mask()
1415 MVPP2_ISR_RX_TX_MASK_REG(port->id), 0); in mvpp2_interrupts_mask()
1416 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_mask()
1417 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), 0); in mvpp2_interrupts_mask()
1431 if (cpu >= port->priv->nthreads) in mvpp2_interrupts_unmask()
1434 thread = mvpp2_cpu_to_thread(port->priv, cpu); in mvpp2_interrupts_unmask()
1437 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version); in mvpp2_interrupts_unmask()
1438 if (port->has_tx_irqs) in mvpp2_interrupts_unmask()
1441 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_unmask()
1442 MVPP2_ISR_RX_TX_MASK_REG(port->id), val); in mvpp2_interrupts_unmask()
1443 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_unmask()
1444 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), in mvpp2_interrupts_unmask()
1454 if (port->priv->hw_version == MVPP21) in mvpp2_shared_interrupt_mask_unmask()
1462 for (i = 0; i < port->nqvecs; i++) { in mvpp2_shared_interrupt_mask_unmask()
1463 struct mvpp2_queue_vector *v = port->qvecs + i; in mvpp2_shared_interrupt_mask_unmask()
1465 if (v->type != MVPP2_QUEUE_VECTOR_SHARED) in mvpp2_shared_interrupt_mask_unmask()
1468 mvpp2_thread_write(port->priv, v->sw_thread_id, in mvpp2_shared_interrupt_mask_unmask()
1469 MVPP2_ISR_RX_TX_MASK_REG(port->id), val); in mvpp2_shared_interrupt_mask_unmask()
1470 mvpp2_thread_write(port->priv, v->sw_thread_id, in mvpp2_shared_interrupt_mask_unmask()
1471 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), in mvpp2_shared_interrupt_mask_unmask()
1479 return port->gop_id == 0; in mvpp2_port_supports_xlg()
1484 return !(port->priv->hw_version >= MVPP22 && port->gop_id == 0); in mvpp2_port_supports_rgmii()
1507 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_rgmii()
1510 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init_rgmii()
1512 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init_rgmii()
1514 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val); in mvpp22_gop_init_rgmii()
1515 if (port->gop_id == 2) in mvpp22_gop_init_rgmii()
1517 else if (port->gop_id == 3) in mvpp22_gop_init_rgmii()
1519 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val); in mvpp22_gop_init_rgmii()
1524 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_sgmii()
1527 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init_sgmii()
1530 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init_sgmii()
1532 if (port->gop_id > 1) { in mvpp22_gop_init_sgmii()
1533 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val); in mvpp22_gop_init_sgmii()
1534 if (port->gop_id == 2) in mvpp22_gop_init_sgmii()
1536 else if (port->gop_id == 3) in mvpp22_gop_init_sgmii()
1538 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val); in mvpp22_gop_init_sgmii()
1544 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_10gkr()
1545 void __iomem *mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_gop_init_10gkr()
1546 void __iomem *xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_gop_init_10gkr()
1567 struct mvpp2 *priv = port->priv; in mvpp22_gop_fca_enable_periodic()
1568 void __iomem *fca = priv->iface_base + MVPP22_FCA_BASE(port->gop_id); in mvpp22_gop_fca_enable_periodic()
1580 struct mvpp2 *priv = port->priv; in mvpp22_gop_fca_set_timer()
1581 void __iomem *fca = priv->iface_base + MVPP22_FCA_BASE(port->gop_id); in mvpp22_gop_fca_set_timer()
1598 timer = (port->priv->tclk / (USEC_PER_SEC * FC_CLK_DIVIDER)) in mvpp22_gop_fca_set_periodic_timer()
1610 struct mvpp2 *priv = port->priv; in mvpp22_gop_init()
1613 if (!priv->sysctrl_base) in mvpp22_gop_init()
1616 switch (port->phy_interface) { in mvpp22_gop_init()
1639 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL1, &val); in mvpp22_gop_init()
1640 val |= GENCONF_PORT_CTRL1_RESET(port->gop_id) | in mvpp22_gop_init()
1641 GENCONF_PORT_CTRL1_EN(port->gop_id); in mvpp22_gop_init()
1642 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL1, val); in mvpp22_gop_init()
1644 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init()
1646 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init()
1648 regmap_read(priv->sysctrl_base, GENCONF_SOFT_RESET1, &val); in mvpp22_gop_init()
1650 regmap_write(priv->sysctrl_base, GENCONF_SOFT_RESET1, val); in mvpp22_gop_init()
1658 netdev_err(port->dev, "Invalid port configuration\n"); in mvpp22_gop_init()
1659 return -EINVAL; in mvpp22_gop_init()
1666 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_unmask_irq()
1667 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_unmask_irq()
1668 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_unmask_irq()
1670 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_unmask_irq()
1672 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_unmask_irq()
1677 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_unmask_irq()
1678 if (mvpp2_is_xlg(port->phy_interface)) in mvpp22_gop_unmask_irq()
1682 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_unmask_irq()
1691 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_mask_irq()
1694 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_mask_irq()
1697 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_mask_irq()
1698 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_mask_irq()
1699 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_mask_irq()
1700 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_mask_irq()
1702 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_mask_irq()
1710 mvpp2_modify(port->base + MVPP22_GMAC_INT_SUM_MASK, in mvpp22_gop_setup_irq()
1714 if (port->phylink || in mvpp22_gop_setup_irq()
1715 phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_setup_irq()
1716 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_setup_irq()
1717 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_setup_irq()
1718 val = readl(port->base + MVPP22_GMAC_INT_MASK); in mvpp22_gop_setup_irq()
1720 writel(val, port->base + MVPP22_GMAC_INT_MASK); in mvpp22_gop_setup_irq()
1724 val = readl(port->base + MVPP22_XLG_INT_MASK); in mvpp22_gop_setup_irq()
1726 writel(val, port->base + MVPP22_XLG_INT_MASK); in mvpp22_gop_setup_irq()
1728 mvpp2_modify(port->base + MVPP22_XLG_EXT_INT_MASK, in mvpp22_gop_setup_irq()
1744 * "PPv2 (2500BaseX) - COMPHY (2500SGMII)" are valid.
1750 if (!port->comphy) in mvpp22_comphy_init()
1753 ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET, in mvpp22_comphy_init()
1754 port->phy_interface); in mvpp22_comphy_init()
1758 return phy_power_on(port->comphy); in mvpp22_comphy_init()
1766 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_enable()
1767 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_enable()
1770 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_enable()
1772 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_enable()
1775 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_enable()
1784 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_disable()
1785 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_disable()
1787 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_disable()
1790 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_disable()
1792 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_disable()
1800 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG) & in mvpp2_port_periodic_xon_disable()
1802 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_periodic_xon_disable()
1811 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_loopback_set()
1813 if (state->speed == 1000) in mvpp2_port_loopback_set()
1818 if (phy_interface_mode_is_8023z(state->interface) || in mvpp2_port_loopback_set()
1819 state->interface == PHY_INTERFACE_MODE_SGMII) in mvpp2_port_loopback_set()
1824 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_loopback_set()
1848 val = readl(port->stats_base + counter->offset); in mvpp2_read_count()
1849 if (counter->reg_is_64b) in mvpp2_read_count()
1850 val += (u64)readl(port->stats_base + counter->offset + 4) << 32; in mvpp2_read_count()
1966 for (q = 0; q < port->ntxqs; q++) { in mvpp2_ethtool_get_strings()
1974 for (q = 0; q < port->nrxqs; q++) { in mvpp2_ethtool_get_strings()
2007 cpu_stats = per_cpu_ptr(port->stats, cpu); in mvpp2_get_xdp_stats()
2009 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); in mvpp2_get_xdp_stats()
2010 xdp_redirect = cpu_stats->xdp_redirect; in mvpp2_get_xdp_stats()
2011 xdp_pass = cpu_stats->xdp_pass; in mvpp2_get_xdp_stats()
2012 xdp_drop = cpu_stats->xdp_drop; in mvpp2_get_xdp_stats()
2013 xdp_xmit = cpu_stats->xdp_xmit; in mvpp2_get_xdp_stats()
2014 xdp_xmit_err = cpu_stats->xdp_xmit_err; in mvpp2_get_xdp_stats()
2015 xdp_tx = cpu_stats->xdp_tx; in mvpp2_get_xdp_stats()
2016 xdp_tx_err = cpu_stats->xdp_tx_err; in mvpp2_get_xdp_stats()
2017 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); in mvpp2_get_xdp_stats()
2019 xdp_stats->xdp_redirect += xdp_redirect; in mvpp2_get_xdp_stats()
2020 xdp_stats->xdp_pass += xdp_pass; in mvpp2_get_xdp_stats()
2021 xdp_stats->xdp_drop += xdp_drop; in mvpp2_get_xdp_stats()
2022 xdp_stats->xdp_xmit += xdp_xmit; in mvpp2_get_xdp_stats()
2023 xdp_stats->xdp_xmit_err += xdp_xmit_err; in mvpp2_get_xdp_stats()
2024 xdp_stats->xdp_tx += xdp_tx; in mvpp2_get_xdp_stats()
2025 xdp_stats->xdp_tx_err += xdp_tx_err; in mvpp2_get_xdp_stats()
2036 pstats = port->ethtool_stats; in mvpp2_read_stats()
2042 *pstats++ += mvpp2_read(port->priv, in mvpp2_read_stats()
2044 4 * port->id); in mvpp2_read_stats()
2046 for (q = 0; q < port->ntxqs; q++) in mvpp2_read_stats()
2048 *pstats++ += mvpp2_read_index(port->priv, in mvpp2_read_stats()
2049 MVPP22_CTRS_TX_CTR(port->id, q), in mvpp2_read_stats()
2053 * driver's. We need to add the port->first_rxq offset. in mvpp2_read_stats()
2055 for (q = 0; q < port->nrxqs; q++) in mvpp2_read_stats()
2057 *pstats++ += mvpp2_read_index(port->priv, in mvpp2_read_stats()
2058 port->first_rxq + q, in mvpp2_read_stats()
2067 switch (s->offset) { in mvpp2_read_stats()
2099 mutex_lock(&port->gather_stats_lock); in mvpp2_gather_hw_statistics()
2106 cancel_delayed_work(&port->stats_work); in mvpp2_gather_hw_statistics()
2107 queue_delayed_work(port->priv->stats_queue, &port->stats_work, in mvpp2_gather_hw_statistics()
2110 mutex_unlock(&port->gather_stats_lock); in mvpp2_gather_hw_statistics()
2121 mvpp2_gather_hw_statistics(&port->stats_work.work); in mvpp2_ethtool_get_stats()
2123 mutex_lock(&port->gather_stats_lock); in mvpp2_ethtool_get_stats()
2124 memcpy(data, port->ethtool_stats, in mvpp2_ethtool_get_stats()
2125 sizeof(u64) * MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs)); in mvpp2_ethtool_get_stats()
2126 mutex_unlock(&port->gather_stats_lock); in mvpp2_ethtool_get_stats()
2134 return MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs); in mvpp2_ethtool_get_sset_count()
2136 return -EOPNOTSUPP; in mvpp2_ethtool_get_sset_count()
2143 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG) | in mvpp2_mac_reset_assert()
2145 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_mac_reset_assert()
2147 if (port->priv->hw_version >= MVPP22 && port->gop_id == 0) { in mvpp2_mac_reset_assert()
2148 val = readl(port->base + MVPP22_XLG_CTRL0_REG) & in mvpp2_mac_reset_assert()
2150 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_reset_assert()
2156 struct mvpp2 *priv = port->priv; in mvpp22_pcs_reset_assert()
2160 if (port->priv->hw_version == MVPP21 || port->gop_id != 0) in mvpp22_pcs_reset_assert()
2163 mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_pcs_reset_assert()
2164 xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_pcs_reset_assert()
2177 struct mvpp2 *priv = port->priv; in mvpp22_pcs_reset_deassert()
2181 if (port->priv->hw_version == MVPP21 || port->gop_id != 0) in mvpp22_pcs_reset_deassert()
2184 mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_pcs_reset_deassert()
2185 xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_pcs_reset_deassert()
2187 switch (port->phy_interface) { in mvpp22_pcs_reset_deassert()
2210 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_max_rx_size_set()
2212 val |= (((port->pkt_size - MVPP2_MH_SIZE) / 2) << in mvpp2_gmac_max_rx_size_set()
2214 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_max_rx_size_set()
2222 val = readl(port->base + MVPP22_XLG_CTRL1_REG); in mvpp2_xlg_max_rx_size_set()
2224 val |= ((port->pkt_size - MVPP2_MH_SIZE) / 2) << in mvpp2_xlg_max_rx_size_set()
2226 writel(val, port->base + MVPP22_XLG_CTRL1_REG); in mvpp2_xlg_max_rx_size_set()
2234 if (port->priv->hw_version == MVPP21) { in mvpp2_defaults_set()
2236 val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG); in mvpp2_defaults_set()
2239 val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2); in mvpp2_defaults_set()
2240 writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG); in mvpp2_defaults_set()
2245 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, in mvpp2_defaults_set()
2247 mvpp2_write(port->priv, MVPP2_TXP_SCHED_CMD_1_REG, 0); in mvpp2_defaults_set()
2249 /* Set TXQ scheduling to Round-Robin */ in mvpp2_defaults_set()
2250 mvpp2_write(port->priv, MVPP2_TXP_SCHED_FIXED_PRIO_REG, 0); in mvpp2_defaults_set()
2254 mvpp2_write(port->priv, in mvpp2_defaults_set()
2260 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PERIOD_REG, in mvpp2_defaults_set()
2261 port->priv->tclk / USEC_PER_SEC); in mvpp2_defaults_set()
2262 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_REFILL_REG); in mvpp2_defaults_set()
2266 mvpp2_write(port->priv, MVPP2_TXP_SCHED_REFILL_REG, val); in mvpp2_defaults_set()
2268 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val); in mvpp2_defaults_set()
2271 mvpp2_write(port->priv, MVPP2_RX_CTRL_REG(port->id), in mvpp2_defaults_set()
2276 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_defaults_set()
2277 queue = port->rxqs[lrxq]->id; in mvpp2_defaults_set()
2278 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_defaults_set()
2281 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_defaults_set()
2294 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_ingress_enable()
2295 queue = port->rxqs[lrxq]->id; in mvpp2_ingress_enable()
2296 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_ingress_enable()
2298 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_ingress_enable()
2307 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_ingress_disable()
2308 queue = port->rxqs[lrxq]->id; in mvpp2_ingress_disable()
2309 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_ingress_disable()
2311 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_ingress_disable()
2316 * - HW starts take descriptors from DRAM
2326 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_egress_enable()
2327 struct mvpp2_tx_queue *txq = port->txqs[queue]; in mvpp2_egress_enable()
2329 if (txq->descs) in mvpp2_egress_enable()
2333 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_egress_enable()
2334 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, qmap); in mvpp2_egress_enable()
2338 * - HW doesn't take descriptors from DRAM
2347 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_egress_disable()
2348 reg_data = (mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG)) & in mvpp2_egress_disable()
2351 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, in mvpp2_egress_disable()
2358 netdev_warn(port->dev, in mvpp2_egress_disable()
2369 reg_data = mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG); in mvpp2_egress_disable()
2379 u32 val = mvpp2_read(port->priv, MVPP2_RXQ_STATUS_REG(rxq_id)); in mvpp2_rxq_received()
2396 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_UPDATE_REG(rxq_id), val); in mvpp2_rxq_status_update()
2403 int rx_desc = rxq->next_desc_to_proc; in mvpp2_rxq_next_desc_get()
2405 rxq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(rxq, rx_desc); in mvpp2_rxq_next_desc_get()
2406 prefetch(rxq->descs + rxq->next_desc_to_proc); in mvpp2_rxq_next_desc_get()
2407 return rxq->descs + rx_desc; in mvpp2_rxq_next_desc_get()
2419 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_offset_set()
2426 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_offset_set()
2435 int tx_desc = txq->next_desc_to_proc; in mvpp2_txq_next_desc_get()
2437 txq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(txq, tx_desc); in mvpp2_txq_next_desc_get()
2438 return txq->descs + tx_desc; in mvpp2_txq_next_desc_get()
2448 /* aggregated access - relevant TXQ number is written in TX desc */ in mvpp2_aggr_txq_pend_desc_add()
2449 mvpp2_thread_write(port->priv, in mvpp2_aggr_txq_pend_desc_add()
2450 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_aggr_txq_pend_desc_add()
2463 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) { in mvpp2_aggr_desc_num_check()
2466 mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_aggr_desc_num_check()
2467 u32 val = mvpp2_read_relaxed(port->priv, in mvpp2_aggr_desc_num_check()
2470 aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK; in mvpp2_aggr_desc_num_check()
2472 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) in mvpp2_aggr_desc_num_check()
2473 return -ENOMEM; in mvpp2_aggr_desc_num_check()
2487 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_txq_alloc_reserved_desc()
2488 struct mvpp2 *priv = port->priv; in mvpp2_txq_alloc_reserved_desc()
2491 val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num; in mvpp2_txq_alloc_reserved_desc()
2510 if (txq_pcpu->reserved_num >= num) in mvpp2_txq_reserved_desc_num_proc()
2519 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_reserved_desc_num_proc()
2522 txq_pcpu_aux = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_reserved_desc_num_proc()
2523 desc_count += txq_pcpu_aux->count; in mvpp2_txq_reserved_desc_num_proc()
2524 desc_count += txq_pcpu_aux->reserved_num; in mvpp2_txq_reserved_desc_num_proc()
2527 req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num); in mvpp2_txq_reserved_desc_num_proc()
2531 (txq->size - (MVPP2_MAX_THREADS * MVPP2_CPU_DESC_CHUNK))) in mvpp2_txq_reserved_desc_num_proc()
2532 return -ENOMEM; in mvpp2_txq_reserved_desc_num_proc()
2534 txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req); in mvpp2_txq_reserved_desc_num_proc()
2537 if (txq_pcpu->reserved_num < num) in mvpp2_txq_reserved_desc_num_proc()
2538 return -ENOMEM; in mvpp2_txq_reserved_desc_num_proc()
2547 if (txq->next_desc_to_proc == 0) in mvpp2_txq_desc_put()
2548 txq->next_desc_to_proc = txq->last_desc - 1; in mvpp2_txq_desc_put()
2550 txq->next_desc_to_proc--; in mvpp2_txq_desc_put()
2588 * Per-thread access
2600 val = mvpp2_thread_read_relaxed(port->priv, in mvpp2_txq_sent_desc_proc()
2601 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_txq_sent_desc_proc()
2602 MVPP2_TXQ_SENT_REG(txq->id)); in mvpp2_txq_sent_desc_proc()
2617 if (smp_processor_id() >= port->priv->nthreads) in mvpp2_txq_sent_counter_clear()
2620 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_txq_sent_counter_clear()
2621 int id = port->txqs[queue]->id; in mvpp2_txq_sent_counter_clear()
2623 mvpp2_thread_read(port->priv, in mvpp2_txq_sent_counter_clear()
2624 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_txq_sent_counter_clear()
2635 mtu = port->pkt_size * 8; in mvpp2_txp_max_tx_size_set()
2644 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_txp_max_tx_size_set()
2647 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_MTU_REG); in mvpp2_txp_max_tx_size_set()
2650 mvpp2_write(port->priv, MVPP2_TXP_SCHED_MTU_REG, val); in mvpp2_txp_max_tx_size_set()
2653 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG); in mvpp2_txp_max_tx_size_set()
2659 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val); in mvpp2_txp_max_tx_size_set()
2662 for (txq = 0; txq < port->ntxqs; txq++) { in mvpp2_txp_max_tx_size_set()
2663 val = mvpp2_read(port->priv, in mvpp2_txp_max_tx_size_set()
2671 mvpp2_write(port->priv, in mvpp2_txp_max_tx_size_set()
2678 /* Set the number of non-occupied descriptors threshold */
2684 mvpp2_write(port->priv, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_set_rxq_free_tresh()
2686 val = mvpp2_read(port->priv, MVPP2_RXQ_THRESH_REG); in mvpp2_set_rxq_free_tresh()
2689 mvpp2_write(port->priv, MVPP2_RXQ_THRESH_REG, val); in mvpp2_set_rxq_free_tresh()
2698 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rx_pkts_coal_set()
2700 if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK) in mvpp2_rx_pkts_coal_set()
2701 rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK; in mvpp2_rx_pkts_coal_set()
2703 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rx_pkts_coal_set()
2704 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_THRESH_REG, in mvpp2_rx_pkts_coal_set()
2705 rxq->pkts_coal); in mvpp2_rx_pkts_coal_set()
2717 if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK) in mvpp2_tx_pkts_coal_set()
2718 txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK; in mvpp2_tx_pkts_coal_set()
2720 val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET); in mvpp2_tx_pkts_coal_set()
2721 /* PKT-coalescing registers are per-queue + per-thread */ in mvpp2_tx_pkts_coal_set()
2723 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_tx_pkts_coal_set()
2724 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_THRESH_REG, val); in mvpp2_tx_pkts_coal_set()
2750 unsigned long freq = port->priv->tclk; in mvpp2_rx_time_coal_set()
2751 u32 val = mvpp2_usec_to_cycles(rxq->time_coal, freq); in mvpp2_rx_time_coal_set()
2754 rxq->time_coal = in mvpp2_rx_time_coal_set()
2757 /* re-evaluate to get actual register value */ in mvpp2_rx_time_coal_set()
2758 val = mvpp2_usec_to_cycles(rxq->time_coal, freq); in mvpp2_rx_time_coal_set()
2761 mvpp2_write(port->priv, MVPP2_ISR_RX_THRESHOLD_REG(rxq->id), val); in mvpp2_rx_time_coal_set()
2766 unsigned long freq = port->priv->tclk; in mvpp2_tx_time_coal_set()
2767 u32 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq); in mvpp2_tx_time_coal_set()
2770 port->tx_time_coal = in mvpp2_tx_time_coal_set()
2773 /* re-evaluate to get actual register value */ in mvpp2_tx_time_coal_set()
2774 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq); in mvpp2_tx_time_coal_set()
2777 mvpp2_write(port->priv, MVPP2_ISR_TX_THRESHOLD_REG(port->id), val); in mvpp2_tx_time_coal_set()
2794 txq_pcpu->buffs + txq_pcpu->txq_get_index; in mvpp2_txq_bufs_free()
2796 if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma) && in mvpp2_txq_bufs_free()
2797 tx_buf->type != MVPP2_TYPE_XDP_TX) in mvpp2_txq_bufs_free()
2798 dma_unmap_single(port->dev->dev.parent, tx_buf->dma, in mvpp2_txq_bufs_free()
2799 tx_buf->size, DMA_TO_DEVICE); in mvpp2_txq_bufs_free()
2800 if (tx_buf->type == MVPP2_TYPE_SKB && tx_buf->skb) in mvpp2_txq_bufs_free()
2801 dev_kfree_skb_any(tx_buf->skb); in mvpp2_txq_bufs_free()
2802 else if (tx_buf->type == MVPP2_TYPE_XDP_TX || in mvpp2_txq_bufs_free()
2803 tx_buf->type == MVPP2_TYPE_XDP_NDO) in mvpp2_txq_bufs_free()
2804 xdp_return_frame_bulk(tx_buf->xdpf, &bq); in mvpp2_txq_bufs_free()
2816 int queue = fls(cause) - 1; in mvpp2_get_rx_queue()
2818 return port->rxqs[queue]; in mvpp2_get_rx_queue()
2824 int queue = fls(cause) - 1; in mvpp2_get_tx_queue()
2826 return port->txqs[queue]; in mvpp2_get_tx_queue()
2833 struct netdev_queue *nq = netdev_get_tx_queue(port->dev, txq->log_id); in mvpp2_txq_done()
2836 if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id())) in mvpp2_txq_done()
2837 netdev_err(port->dev, "wrong cpu on the end of Tx processing\n"); in mvpp2_txq_done()
2844 txq_pcpu->count -= tx_done; in mvpp2_txq_done()
2847 if (txq_pcpu->count <= txq_pcpu->wake_threshold) in mvpp2_txq_done()
2863 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx_done()
2865 if (txq_pcpu->count) { in mvpp2_tx_done()
2867 tx_todo += txq_pcpu->count; in mvpp2_tx_done()
2870 cause &= ~(1 << txq->log_id); in mvpp2_tx_done()
2885 aggr_txq->descs = dma_alloc_coherent(&pdev->dev, in mvpp2_aggr_txq_init()
2887 &aggr_txq->descs_dma, GFP_KERNEL); in mvpp2_aggr_txq_init()
2888 if (!aggr_txq->descs) in mvpp2_aggr_txq_init()
2889 return -ENOMEM; in mvpp2_aggr_txq_init()
2891 aggr_txq->last_desc = MVPP2_AGGR_TXQ_SIZE - 1; in mvpp2_aggr_txq_init()
2894 aggr_txq->next_desc_to_proc = mvpp2_read(priv, in mvpp2_aggr_txq_init()
2900 if (priv->hw_version == MVPP21) in mvpp2_aggr_txq_init()
2901 txq_dma = aggr_txq->descs_dma; in mvpp2_aggr_txq_init()
2903 txq_dma = aggr_txq->descs_dma >> in mvpp2_aggr_txq_init()
2917 struct mvpp2 *priv = port->priv; in mvpp2_rxq_init()
2922 rxq->size = port->rx_ring_size; in mvpp2_rxq_init()
2925 rxq->descs = dma_alloc_coherent(port->dev->dev.parent, in mvpp2_rxq_init()
2926 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_init()
2927 &rxq->descs_dma, GFP_KERNEL); in mvpp2_rxq_init()
2928 if (!rxq->descs) in mvpp2_rxq_init()
2929 return -ENOMEM; in mvpp2_rxq_init()
2931 rxq->last_desc = rxq->size - 1; in mvpp2_rxq_init()
2933 /* Zero occupied and non-occupied counters - direct access */ in mvpp2_rxq_init()
2934 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0); in mvpp2_rxq_init()
2936 /* Set Rx descriptors queue starting address - indirect access */ in mvpp2_rxq_init()
2937 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rxq_init()
2938 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rxq_init()
2939 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_init()
2940 rxq_dma = rxq->descs_dma; in mvpp2_rxq_init()
2942 rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS; in mvpp2_rxq_init()
2943 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma); in mvpp2_rxq_init()
2944 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, rxq->size); in mvpp2_rxq_init()
2945 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_INDEX_REG, 0); in mvpp2_rxq_init()
2949 mvpp2_rxq_offset_set(port, rxq->id, MVPP2_SKB_HEADROOM); in mvpp2_rxq_init()
2959 mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); in mvpp2_rxq_init()
2961 if (priv->percpu_pools) { in mvpp2_rxq_init()
2962 err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0); in mvpp2_rxq_init()
2966 err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0); in mvpp2_rxq_init()
2971 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_short, in mvpp2_rxq_init()
2973 priv->page_pool[rxq->logic_rxq]); in mvpp2_rxq_init()
2977 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_long, in mvpp2_rxq_init()
2979 priv->page_pool[rxq->logic_rxq + in mvpp2_rxq_init()
2980 port->nrxqs]); in mvpp2_rxq_init()
2988 xdp_rxq_info_unreg_mem_model(&rxq->xdp_rxq_short); in mvpp2_rxq_init()
2990 xdp_rxq_info_unreg(&rxq->xdp_rxq_long); in mvpp2_rxq_init()
2992 xdp_rxq_info_unreg(&rxq->xdp_rxq_short); in mvpp2_rxq_init()
2994 dma_free_coherent(port->dev->dev.parent, in mvpp2_rxq_init()
2995 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_init()
2996 rxq->descs, rxq->descs_dma); in mvpp2_rxq_init()
3006 rx_received = mvpp2_rxq_received(port, rxq->id); in mvpp2_rxq_drop_pkts()
3022 mvpp2_rxq_status_update(port, rxq->id, rx_received, rx_received); in mvpp2_rxq_drop_pkts()
3031 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_short)) in mvpp2_rxq_deinit()
3032 xdp_rxq_info_unreg(&rxq->xdp_rxq_short); in mvpp2_rxq_deinit()
3034 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_long)) in mvpp2_rxq_deinit()
3035 xdp_rxq_info_unreg(&rxq->xdp_rxq_long); in mvpp2_rxq_deinit()
3039 if (rxq->descs) in mvpp2_rxq_deinit()
3040 dma_free_coherent(port->dev->dev.parent, in mvpp2_rxq_deinit()
3041 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_deinit()
3042 rxq->descs, in mvpp2_rxq_deinit()
3043 rxq->descs_dma); in mvpp2_rxq_deinit()
3045 rxq->descs = NULL; in mvpp2_rxq_deinit()
3046 rxq->last_desc = 0; in mvpp2_rxq_deinit()
3047 rxq->next_desc_to_proc = 0; in mvpp2_rxq_deinit()
3048 rxq->descs_dma = 0; in mvpp2_rxq_deinit()
3053 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0); in mvpp2_rxq_deinit()
3054 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rxq_deinit()
3055 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rxq_deinit()
3056 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, 0); in mvpp2_rxq_deinit()
3057 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, 0); in mvpp2_rxq_deinit()
3070 txq->size = port->tx_ring_size; in mvpp2_txq_init()
3073 txq->descs = dma_alloc_coherent(port->dev->dev.parent, in mvpp2_txq_init()
3074 txq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_txq_init()
3075 &txq->descs_dma, GFP_KERNEL); in mvpp2_txq_init()
3076 if (!txq->descs) in mvpp2_txq_init()
3077 return -ENOMEM; in mvpp2_txq_init()
3079 txq->last_desc = txq->size - 1; in mvpp2_txq_init()
3081 /* Set Tx descriptors queue starting address - indirect access */ in mvpp2_txq_init()
3082 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_init()
3083 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_init()
3084 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, in mvpp2_txq_init()
3085 txq->descs_dma); in mvpp2_txq_init()
3086 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, in mvpp2_txq_init()
3087 txq->size & MVPP2_TXQ_DESC_SIZE_MASK); in mvpp2_txq_init()
3088 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_INDEX_REG, 0); in mvpp2_txq_init()
3089 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_RSVD_CLR_REG, in mvpp2_txq_init()
3090 txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET); in mvpp2_txq_init()
3091 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PENDING_REG); in mvpp2_txq_init()
3093 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PENDING_REG, val); in mvpp2_txq_init()
3101 desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) + in mvpp2_txq_init()
3102 (txq->log_id * desc_per_txq); in mvpp2_txq_init()
3104 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, in mvpp2_txq_init()
3109 /* WRR / EJP configuration - indirect access */ in mvpp2_txq_init()
3111 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_txq_init()
3113 val = mvpp2_read(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id)); in mvpp2_txq_init()
3117 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id), val); in mvpp2_txq_init()
3120 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq->log_id), in mvpp2_txq_init()
3123 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_init()
3124 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_init()
3125 txq_pcpu->size = txq->size; in mvpp2_txq_init()
3126 txq_pcpu->buffs = kmalloc_array(txq_pcpu->size, in mvpp2_txq_init()
3127 sizeof(*txq_pcpu->buffs), in mvpp2_txq_init()
3129 if (!txq_pcpu->buffs) in mvpp2_txq_init()
3130 return -ENOMEM; in mvpp2_txq_init()
3132 txq_pcpu->count = 0; in mvpp2_txq_init()
3133 txq_pcpu->reserved_num = 0; in mvpp2_txq_init()
3134 txq_pcpu->txq_put_index = 0; in mvpp2_txq_init()
3135 txq_pcpu->txq_get_index = 0; in mvpp2_txq_init()
3136 txq_pcpu->tso_headers = NULL; in mvpp2_txq_init()
3138 txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS; in mvpp2_txq_init()
3139 txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2; in mvpp2_txq_init()
3141 txq_pcpu->tso_headers = in mvpp2_txq_init()
3142 dma_alloc_coherent(port->dev->dev.parent, in mvpp2_txq_init()
3143 txq_pcpu->size * TSO_HEADER_SIZE, in mvpp2_txq_init()
3144 &txq_pcpu->tso_headers_dma, in mvpp2_txq_init()
3146 if (!txq_pcpu->tso_headers) in mvpp2_txq_init()
3147 return -ENOMEM; in mvpp2_txq_init()
3160 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_deinit()
3161 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_deinit()
3162 kfree(txq_pcpu->buffs); in mvpp2_txq_deinit()
3164 if (txq_pcpu->tso_headers) in mvpp2_txq_deinit()
3165 dma_free_coherent(port->dev->dev.parent, in mvpp2_txq_deinit()
3166 txq_pcpu->size * TSO_HEADER_SIZE, in mvpp2_txq_deinit()
3167 txq_pcpu->tso_headers, in mvpp2_txq_deinit()
3168 txq_pcpu->tso_headers_dma); in mvpp2_txq_deinit()
3170 txq_pcpu->tso_headers = NULL; in mvpp2_txq_deinit()
3173 if (txq->descs) in mvpp2_txq_deinit()
3174 dma_free_coherent(port->dev->dev.parent, in mvpp2_txq_deinit()
3175 txq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_txq_deinit()
3176 txq->descs, txq->descs_dma); in mvpp2_txq_deinit()
3178 txq->descs = NULL; in mvpp2_txq_deinit()
3179 txq->last_desc = 0; in mvpp2_txq_deinit()
3180 txq->next_desc_to_proc = 0; in mvpp2_txq_deinit()
3181 txq->descs_dma = 0; in mvpp2_txq_deinit()
3184 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->log_id), 0); in mvpp2_txq_deinit()
3187 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_deinit()
3188 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_deinit()
3189 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, 0); in mvpp2_txq_deinit()
3190 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, 0); in mvpp2_txq_deinit()
3199 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_clean()
3202 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_clean()
3203 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG); in mvpp2_txq_clean()
3205 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val); in mvpp2_txq_clean()
3213 netdev_warn(port->dev, in mvpp2_txq_clean()
3215 port->id, txq->log_id); in mvpp2_txq_clean()
3221 pending = mvpp2_thread_read(port->priv, thread, in mvpp2_txq_clean()
3227 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val); in mvpp2_txq_clean()
3230 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_clean()
3231 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_clean()
3234 mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count); in mvpp2_txq_clean()
3237 txq_pcpu->count = 0; in mvpp2_txq_clean()
3238 txq_pcpu->txq_put_index = 0; in mvpp2_txq_clean()
3239 txq_pcpu->txq_get_index = 0; in mvpp2_txq_clean()
3250 val = mvpp2_read(port->priv, MVPP2_TX_PORT_FLUSH_REG); in mvpp2_cleanup_txqs()
3253 val |= MVPP2_TX_PORT_FLUSH_MASK(port->id); in mvpp2_cleanup_txqs()
3254 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val); in mvpp2_cleanup_txqs()
3256 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_cleanup_txqs()
3257 txq = port->txqs[queue]; in mvpp2_cleanup_txqs()
3264 val &= ~MVPP2_TX_PORT_FLUSH_MASK(port->id); in mvpp2_cleanup_txqs()
3265 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val); in mvpp2_cleanup_txqs()
3273 for (queue = 0; queue < port->nrxqs; queue++) in mvpp2_cleanup_rxqs()
3274 mvpp2_rxq_deinit(port, port->rxqs[queue]); in mvpp2_cleanup_rxqs()
3276 if (port->tx_fc) in mvpp2_cleanup_rxqs()
3285 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_setup_rxqs()
3286 err = mvpp2_rxq_init(port, port->rxqs[queue]); in mvpp2_setup_rxqs()
3291 if (port->tx_fc) in mvpp2_setup_rxqs()
3307 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_setup_txqs()
3308 txq = port->txqs[queue]; in mvpp2_setup_txqs()
3315 netif_set_xps_queue(port->dev, cpumask_of(queue), queue); in mvpp2_setup_txqs()
3318 if (port->has_tx_irqs) { in mvpp2_setup_txqs()
3320 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_setup_txqs()
3321 txq = port->txqs[queue]; in mvpp2_setup_txqs()
3334 /* The callback for per-port interrupt */
3341 napi_schedule(&qv->napi); in mvpp2_isr()
3355 ptp_q = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_isr_handle_ptp_queue()
3357 ptp_q += MVPP22_PTP_TX_Q1_R0 - MVPP22_PTP_TX_Q0_R0; in mvpp2_isr_handle_ptp_queue()
3359 queue = &port->tx_hwtstamp_queue[nq]; in mvpp2_isr_handle_ptp_queue()
3371 skb = queue->skb[id]; in mvpp2_isr_handle_ptp_queue()
3372 queue->skb[id] = NULL; in mvpp2_isr_handle_ptp_queue()
3376 mvpp22_tai_tstamp(port->priv->tai, ts, &shhwtstamps); in mvpp2_isr_handle_ptp_queue()
3388 ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_isr_handle_ptp()
3398 struct net_device *dev = port->dev; in mvpp2_isr_handle_link()
3400 if (port->phylink) { in mvpp2_isr_handle_link()
3401 phylink_mac_change(port->phylink, link); in mvpp2_isr_handle_link()
3430 val = readl(port->base + MVPP22_XLG_INT_STAT); in mvpp2_isr_handle_xlg()
3432 val = readl(port->base + MVPP22_XLG_STATUS); in mvpp2_isr_handle_xlg()
3443 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp2_isr_handle_gmac_internal()
3444 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp2_isr_handle_gmac_internal()
3445 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_isr_handle_gmac_internal()
3446 val = readl(port->base + MVPP22_GMAC_INT_STAT); in mvpp2_isr_handle_gmac_internal()
3448 val = readl(port->base + MVPP2_GMAC_STATUS0); in mvpp2_isr_handle_gmac_internal()
3455 /* Per-port interrupt for link status changes */
3464 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_isr()
3466 val = readl(port->base + MVPP22_XLG_EXT_INT_STAT); in mvpp2_port_isr()
3475 val = readl(port->base + MVPP22_GMAC_INT_SUM_STAT); in mvpp2_port_isr()
3494 dev = port_pcpu->dev; in mvpp2_hr_timer_cb()
3499 port_pcpu->timer_scheduled = false; in mvpp2_hr_timer_cb()
3503 cause = (1 << port->ntxqs) - 1; in mvpp2_hr_timer_cb()
3505 mvpp2_cpu_to_thread(port->priv, smp_processor_id())); in mvpp2_hr_timer_cb()
3508 if (tx_todo && !port_pcpu->timer_scheduled) { in mvpp2_hr_timer_cb()
3509 port_pcpu->timer_scheduled = true; in mvpp2_hr_timer_cb()
3510 hrtimer_forward_now(&port_pcpu->tx_done_timer, in mvpp2_hr_timer_cb()
3540 netdev_err(port->dev, in mvpp2_rx_error()
3571 return -ENOMEM; in mvpp2_rx_refill()
3581 if (skb->ip_summed == CHECKSUM_PARTIAL) { in mvpp2_skb_tx_csum()
3590 ip_hdr_len = ip4h->ihl; in mvpp2_skb_tx_csum()
3591 l4_proto = ip4h->protocol; in mvpp2_skb_tx_csum()
3598 l4_proto = ip6h->nexthdr; in mvpp2_skb_tx_csum()
3612 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_xdp_finish_tx()
3618 txq = port->txqs[txq_id]; in mvpp2_xdp_finish_tx()
3619 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_xdp_finish_tx()
3620 nq = netdev_get_tx_queue(port->dev, txq_id); in mvpp2_xdp_finish_tx()
3621 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_xdp_finish_tx()
3623 txq_pcpu->reserved_num -= nxmit; in mvpp2_xdp_finish_tx()
3624 txq_pcpu->count += nxmit; in mvpp2_xdp_finish_tx()
3625 aggr_txq->count += nxmit; in mvpp2_xdp_finish_tx()
3631 if (txq_pcpu->count >= txq_pcpu->stop_threshold) in mvpp2_xdp_finish_tx()
3635 if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal) in mvpp2_xdp_finish_tx()
3643 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_xdp_submit_frame()
3654 txq = port->txqs[txq_id]; in mvpp2_xdp_submit_frame()
3655 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_xdp_submit_frame()
3656 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_xdp_submit_frame()
3667 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_xdp_submit_frame()
3668 mvpp2_txdesc_size_set(port, tx_desc, xdpf->len); in mvpp2_xdp_submit_frame()
3672 dma_addr = dma_map_single(port->dev->dev.parent, xdpf->data, in mvpp2_xdp_submit_frame()
3673 xdpf->len, DMA_TO_DEVICE); in mvpp2_xdp_submit_frame()
3675 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) { in mvpp2_xdp_submit_frame()
3684 struct page *page = virt_to_page(xdpf->data); in mvpp2_xdp_submit_frame()
3687 sizeof(*xdpf) + xdpf->headroom; in mvpp2_xdp_submit_frame()
3688 dma_sync_single_for_device(port->dev->dev.parent, dma_addr, in mvpp2_xdp_submit_frame()
3689 xdpf->len, DMA_BIDIRECTIONAL); in mvpp2_xdp_submit_frame()
3706 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats); in mvpp2_xdp_xmit_back()
3718 txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2); in mvpp2_xdp_xmit_back()
3722 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit_back()
3723 stats->tx_bytes += xdpf->len; in mvpp2_xdp_xmit_back()
3724 stats->tx_packets++; in mvpp2_xdp_xmit_back()
3725 stats->xdp_tx++; in mvpp2_xdp_xmit_back()
3726 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit_back()
3728 mvpp2_xdp_finish_tx(port, txq_id, 1, xdpf->len); in mvpp2_xdp_xmit_back()
3730 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit_back()
3731 stats->xdp_tx_err++; in mvpp2_xdp_xmit_back()
3732 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit_back()
3748 if (unlikely(test_bit(0, &port->state))) in mvpp2_xdp_xmit()
3749 return -ENETDOWN; in mvpp2_xdp_xmit()
3752 return -EINVAL; in mvpp2_xdp_xmit()
3757 txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2); in mvpp2_xdp_xmit()
3764 nxmit_byte += frames[i]->len; in mvpp2_xdp_xmit()
3771 stats = this_cpu_ptr(port->stats); in mvpp2_xdp_xmit()
3772 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit()
3773 stats->tx_bytes += nxmit_byte; in mvpp2_xdp_xmit()
3774 stats->tx_packets += nxmit; in mvpp2_xdp_xmit()
3775 stats->xdp_xmit += nxmit; in mvpp2_xdp_xmit()
3776 stats->xdp_xmit_err += num_frame - nxmit; in mvpp2_xdp_xmit()
3777 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit()
3791 len = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM; in mvpp2_run_xdp()
3795 sync = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM; in mvpp2_run_xdp()
3800 stats->xdp_pass++; in mvpp2_run_xdp()
3804 err = xdp_do_redirect(port->dev, xdp, prog); in mvpp2_run_xdp()
3807 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3811 stats->xdp_redirect++; in mvpp2_run_xdp()
3817 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3825 trace_xdp_exception(port->dev, prog, act); in mvpp2_run_xdp()
3828 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3831 stats->xdp_drop++; in mvpp2_run_xdp()
3851 phys_addr_next = le32_to_cpu(buff_hdr->next_phys_addr); in mvpp2_buff_hdr_pool_put()
3852 dma_addr_next = le32_to_cpu(buff_hdr->next_dma_addr); in mvpp2_buff_hdr_pool_put()
3854 if (port->priv->hw_version >= MVPP22) { in mvpp2_buff_hdr_pool_put()
3855 phys_addr_next |= ((u64)buff_hdr->next_phys_addr_high << 32); in mvpp2_buff_hdr_pool_put()
3856 dma_addr_next |= ((u64)buff_hdr->next_dma_addr_high << 32); in mvpp2_buff_hdr_pool_put()
3864 } while (!MVPP2_B_HDR_INFO_IS_LAST(le16_to_cpu(buff_hdr->info))); in mvpp2_buff_hdr_pool_put()
3871 struct net_device *dev = port->dev; in mvpp2_rx()
3880 xdp_prog = READ_ONCE(port->xdp_prog); in mvpp2_rx()
3882 /* Get number of received packets and clamp the to-do */ in mvpp2_rx()
3883 rx_received = mvpp2_rxq_received(port, rxq->id); in mvpp2_rx()
3908 rx_bytes -= MVPP2_MH_SIZE; in mvpp2_rx()
3913 bm_pool = &port->priv->bm_pools[pool]; in mvpp2_rx()
3915 if (port->priv->percpu_pools) { in mvpp2_rx()
3916 pp = port->priv->page_pool[pool]; in mvpp2_rx()
3922 dma_sync_single_for_cpu(dev->dev.parent, dma_addr, in mvpp2_rx()
3941 if (bm_pool->frag_size > PAGE_SIZE) in mvpp2_rx()
3944 frag_size = bm_pool->frag_size; in mvpp2_rx()
3949 if (bm_pool->pkt_size == MVPP2_BM_SHORT_PKT_SIZE) in mvpp2_rx()
3950 xdp_rxq = &rxq->xdp_rxq_short; in mvpp2_rx()
3952 xdp_rxq = &rxq->xdp_rxq_long; in mvpp2_rx()
3965 netdev_err(port->dev, "failed to refill BM pools\n"); in mvpp2_rx()
3977 netdev_warn(port->dev, "skb build failed\n"); in mvpp2_rx()
3985 timestamp = le32_to_cpu(rx_desc->pp22.timestamp); in mvpp2_rx()
3986 mvpp22_tai_tstamp(port->priv->tai, timestamp, in mvpp2_rx()
3992 netdev_err(port->dev, "failed to refill BM pools\n"); in mvpp2_rx()
4000 dma_unmap_single_attrs(dev->dev.parent, dma_addr, in mvpp2_rx()
4001 bm_pool->buf_size, DMA_FROM_DEVICE, in mvpp2_rx()
4009 skb->ip_summed = mvpp2_rx_csum(port, rx_status); in mvpp2_rx()
4010 skb->protocol = eth_type_trans(skb, dev); in mvpp2_rx()
4016 dev->stats.rx_errors++; in mvpp2_rx()
4029 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats); in mvpp2_rx()
4031 u64_stats_update_begin(&stats->syncp); in mvpp2_rx()
4032 stats->rx_packets += ps.rx_packets; in mvpp2_rx()
4033 stats->rx_bytes += ps.rx_bytes; in mvpp2_rx()
4035 stats->xdp_redirect += ps.xdp_redirect; in mvpp2_rx()
4036 stats->xdp_pass += ps.xdp_pass; in mvpp2_rx()
4037 stats->xdp_drop += ps.xdp_drop; in mvpp2_rx()
4038 u64_stats_update_end(&stats->syncp); in mvpp2_rx()
4043 mvpp2_rxq_status_update(port, rxq->id, rx_done, rx_done); in mvpp2_rx()
4052 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in tx_desc_unmap_put()
4053 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in tx_desc_unmap_put()
4060 dma_unmap_single(port->dev->dev.parent, buf_dma_addr, in tx_desc_unmap_put()
4069 if (port->priv->hw_version >= MVPP22) in mvpp2_txdesc_clear_ptp()
4070 desc->pp22.ptp_descriptor &= in mvpp2_txdesc_clear_ptp()
4083 if (port->priv->hw_version == MVPP21 || in mvpp2_tx_hw_tstamp()
4084 port->tx_hwtstamp_type == HWTSTAMP_TX_OFF) in mvpp2_tx_hw_tstamp()
4095 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in mvpp2_tx_hw_tstamp()
4099 queue = &port->tx_hwtstamp_queue[0]; in mvpp2_tx_hw_tstamp()
4108 mtype = hdr->tsmt & 15; in mvpp2_tx_hw_tstamp()
4112 queue = &port->tx_hwtstamp_queue[1]; in mvpp2_tx_hw_tstamp()
4118 i = queue->next; in mvpp2_tx_hw_tstamp()
4119 queue->next = (i + 1) & 31; in mvpp2_tx_hw_tstamp()
4120 if (queue->skb[i]) in mvpp2_tx_hw_tstamp()
4121 dev_kfree_skb_any(queue->skb[i]); in mvpp2_tx_hw_tstamp()
4122 queue->skb[i] = skb_get(skb); in mvpp2_tx_hw_tstamp()
4127 * 3:0 - PTPAction in mvpp2_tx_hw_tstamp()
4128 * 6:4 - PTPPacketFormat in mvpp2_tx_hw_tstamp()
4129 * 7 - PTP_CF_WraparoundCheckEn in mvpp2_tx_hw_tstamp()
4130 * 9:8 - IngressTimestampSeconds[1:0] in mvpp2_tx_hw_tstamp()
4131 * 10 - Reserved in mvpp2_tx_hw_tstamp()
4132 * 11 - MACTimestampingEn in mvpp2_tx_hw_tstamp()
4133 * 17:12 - PTP_TimestampQueueEntryID[5:0] in mvpp2_tx_hw_tstamp()
4134 * 18 - PTPTimestampQueueSelect in mvpp2_tx_hw_tstamp()
4135 * 19 - UDPChecksumUpdateEn in mvpp2_tx_hw_tstamp()
4136 * 27:20 - TimestampOffset in mvpp2_tx_hw_tstamp()
4137 * PTP, NTPTransmit, OWAMP/TWAMP - L3 to PTP header in mvpp2_tx_hw_tstamp()
4138 * NTPTs, Y.1731 - L3 to timestamp entry in mvpp2_tx_hw_tstamp()
4139 * 35:28 - UDP Checksum Offset in mvpp2_tx_hw_tstamp()
4143 tx_desc->pp22.ptp_descriptor &= in mvpp2_tx_hw_tstamp()
4145 tx_desc->pp22.ptp_descriptor |= in mvpp2_tx_hw_tstamp()
4147 tx_desc->pp22.buf_dma_addr_ptp &= cpu_to_le64(~0xffffff0000000000ULL); in mvpp2_tx_hw_tstamp()
4148 tx_desc->pp22.buf_dma_addr_ptp |= cpu_to_le64((ptpdesc >> 12) << 40); in mvpp2_tx_hw_tstamp()
4158 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_tx_frag_process()
4159 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx_frag_process()
4164 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in mvpp2_tx_frag_process()
4165 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in mvpp2_tx_frag_process()
4170 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tx_frag_process()
4173 buf_dma_addr = dma_map_single(port->dev->dev.parent, addr, in mvpp2_tx_frag_process()
4176 if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) { in mvpp2_tx_frag_process()
4183 if (i == (skb_shinfo(skb)->nr_frags - 1)) { in mvpp2_tx_frag_process()
4200 for (i = i - 1; i >= 0; i--) { in mvpp2_tx_frag_process()
4201 tx_desc = txq->descs + i; in mvpp2_tx_frag_process()
4205 return -ENOMEM; in mvpp2_tx_frag_process()
4220 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tso_put_hdr()
4223 addr = txq_pcpu->tso_headers_dma + in mvpp2_tso_put_hdr()
4224 txq_pcpu->txq_put_index * TSO_HEADER_SIZE; in mvpp2_tso_put_hdr()
4245 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tso_put_data()
4248 buf_dma_addr = dma_map_single(dev->dev.parent, tso->data, sz, in mvpp2_tso_put_data()
4250 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) { in mvpp2_tso_put_data()
4252 return -ENOMEM; in mvpp2_tso_put_data()
4288 len = skb->len - hdr_sz; in mvpp2_tx_tso()
4290 int left = min_t(int, skb_shinfo(skb)->gso_size, len); in mvpp2_tx_tso()
4291 char *hdr = txq_pcpu->tso_headers + in mvpp2_tx_tso()
4292 txq_pcpu->txq_put_index * TSO_HEADER_SIZE; in mvpp2_tx_tso()
4294 len -= left; in mvpp2_tx_tso()
4302 left -= sz; in mvpp2_tx_tso()
4315 for (i = descs - 1; i >= 0; i--) { in mvpp2_tx_tso()
4316 struct mvpp2_tx_desc *tx_desc = txq->descs + i; in mvpp2_tx_tso()
4336 thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_tx()
4339 txq = port->txqs[txq_id]; in mvpp2_tx()
4340 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx()
4341 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_tx()
4343 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_tx()
4344 spin_lock_irqsave(&port->tx_lock[thread], flags); in mvpp2_tx()
4350 frags = skb_shinfo(skb)->nr_frags + 1; in mvpp2_tx()
4361 if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) || in mvpp2_tx()
4364 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tx()
4367 buf_dma_addr = dma_map_single(dev->dev.parent, skb->data, in mvpp2_tx()
4369 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) { in mvpp2_tx()
4399 struct mvpp2_pcpu_stats *stats = per_cpu_ptr(port->stats, thread); in mvpp2_tx()
4402 txq_pcpu->reserved_num -= frags; in mvpp2_tx()
4403 txq_pcpu->count += frags; in mvpp2_tx()
4404 aggr_txq->count += frags; in mvpp2_tx()
4410 if (txq_pcpu->count >= txq_pcpu->stop_threshold) in mvpp2_tx()
4413 u64_stats_update_begin(&stats->syncp); in mvpp2_tx()
4414 stats->tx_packets++; in mvpp2_tx()
4415 stats->tx_bytes += skb->len; in mvpp2_tx()
4416 u64_stats_update_end(&stats->syncp); in mvpp2_tx()
4418 dev->stats.tx_dropped++; in mvpp2_tx()
4423 if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal) in mvpp2_tx()
4427 if (!port->has_tx_irqs && txq_pcpu->count <= frags && in mvpp2_tx()
4428 txq_pcpu->count > 0) { in mvpp2_tx()
4429 struct mvpp2_port_pcpu *port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_tx()
4431 if (!port_pcpu->timer_scheduled) { in mvpp2_tx()
4432 port_pcpu->timer_scheduled = true; in mvpp2_tx()
4433 hrtimer_start(&port_pcpu->tx_done_timer, in mvpp2_tx()
4439 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_tx()
4440 spin_unlock_irqrestore(&port->tx_lock[thread], flags); in mvpp2_tx()
4459 struct mvpp2_port *port = netdev_priv(napi->dev); in mvpp2_poll()
4461 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_poll()
4467 * Bits 0-15: each bit indicates received packets on the Rx queue in mvpp2_poll()
4470 * Bits 16-23: each bit indicates transmitted packets on the Tx queue in mvpp2_poll()
4475 cause_rx_tx = mvpp2_thread_read_relaxed(port->priv, qv->sw_thread_id, in mvpp2_poll()
4476 MVPP2_ISR_RX_TX_CAUSE_REG(port->id)); in mvpp2_poll()
4480 mvpp2_cause_error(port->dev, cause_misc); in mvpp2_poll()
4483 mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0); in mvpp2_poll()
4484 mvpp2_thread_write(port->priv, thread, in mvpp2_poll()
4485 MVPP2_ISR_RX_TX_CAUSE_REG(port->id), in mvpp2_poll()
4489 if (port->has_tx_irqs) { in mvpp2_poll()
4493 mvpp2_tx_done(port, cause_tx, qv->sw_thread_id); in mvpp2_poll()
4499 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version); in mvpp2_poll()
4500 cause_rx <<= qv->first_rxq; in mvpp2_poll()
4501 cause_rx |= qv->pending_cause_rx; in mvpp2_poll()
4512 budget -= count; in mvpp2_poll()
4518 cause_rx &= ~(1 << rxq->logic_rxq); in mvpp2_poll()
4528 qv->pending_cause_rx = cause_rx; in mvpp2_poll()
4551 ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG); in mvpp22_mode_reconfigure()
4554 if (mvpp2_is_xlg(port->phy_interface)) in mvpp22_mode_reconfigure()
4559 writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG); in mvpp22_mode_reconfigure()
4562 if (mvpp2_port_supports_xlg(port) && mvpp2_is_xlg(port->phy_interface)) in mvpp22_mode_reconfigure()
4575 for (i = 0; i < port->nqvecs; i++) in mvpp2_start_dev()
4576 napi_enable(&port->qvecs[i].napi); in mvpp2_start_dev()
4581 if (port->priv->hw_version >= MVPP22) in mvpp2_start_dev()
4584 if (port->phylink) { in mvpp2_start_dev()
4585 phylink_start(port->phylink); in mvpp2_start_dev()
4590 netif_tx_start_all_queues(port->dev); in mvpp2_start_dev()
4592 clear_bit(0, &port->state); in mvpp2_start_dev()
4600 set_bit(0, &port->state); in mvpp2_stop_dev()
4605 for (i = 0; i < port->nqvecs; i++) in mvpp2_stop_dev()
4606 napi_disable(&port->qvecs[i].napi); in mvpp2_stop_dev()
4608 if (port->phylink) in mvpp2_stop_dev()
4609 phylink_stop(port->phylink); in mvpp2_stop_dev()
4610 phy_power_off(port->comphy); in mvpp2_stop_dev()
4616 u16 new_rx_pending = ring->rx_pending; in mvpp2_check_ringparam_valid()
4617 u16 new_tx_pending = ring->tx_pending; in mvpp2_check_ringparam_valid()
4619 if (ring->rx_pending == 0 || ring->tx_pending == 0) in mvpp2_check_ringparam_valid()
4620 return -EINVAL; in mvpp2_check_ringparam_valid()
4622 if (ring->rx_pending > MVPP2_MAX_RXD_MAX) in mvpp2_check_ringparam_valid()
4624 else if (ring->rx_pending < MSS_THRESHOLD_START) in mvpp2_check_ringparam_valid()
4626 else if (!IS_ALIGNED(ring->rx_pending, 16)) in mvpp2_check_ringparam_valid()
4627 new_rx_pending = ALIGN(ring->rx_pending, 16); in mvpp2_check_ringparam_valid()
4629 if (ring->tx_pending > MVPP2_MAX_TXD_MAX) in mvpp2_check_ringparam_valid()
4631 else if (!IS_ALIGNED(ring->tx_pending, 32)) in mvpp2_check_ringparam_valid()
4632 new_tx_pending = ALIGN(ring->tx_pending, 32); in mvpp2_check_ringparam_valid()
4640 if (ring->rx_pending != new_rx_pending) { in mvpp2_check_ringparam_valid()
4642 ring->rx_pending, new_rx_pending); in mvpp2_check_ringparam_valid()
4643 ring->rx_pending = new_rx_pending; in mvpp2_check_ringparam_valid()
4646 if (ring->tx_pending != new_tx_pending) { in mvpp2_check_ringparam_valid()
4648 ring->tx_pending, new_tx_pending); in mvpp2_check_ringparam_valid()
4649 ring->tx_pending = new_tx_pending; in mvpp2_check_ringparam_valid()
4659 mac_addr_l = readl(port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp21_get_mac_address()
4660 mac_addr_m = readl(port->priv->lms_base + MVPP2_SRC_ADDR_MIDDLE); in mvpp21_get_mac_address()
4661 mac_addr_h = readl(port->priv->lms_base + MVPP2_SRC_ADDR_HIGH); in mvpp21_get_mac_address()
4674 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_init()
4675 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_init()
4677 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) { in mvpp2_irqs_init()
4678 qv->mask = kzalloc(cpumask_size(), GFP_KERNEL); in mvpp2_irqs_init()
4679 if (!qv->mask) { in mvpp2_irqs_init()
4680 err = -ENOMEM; in mvpp2_irqs_init()
4684 irq_set_status_flags(qv->irq, IRQ_NO_BALANCING); in mvpp2_irqs_init()
4687 err = request_irq(qv->irq, mvpp2_isr, 0, port->dev->name, qv); in mvpp2_irqs_init()
4691 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) { in mvpp2_irqs_init()
4695 if (mvpp2_cpu_to_thread(port->priv, cpu) == in mvpp2_irqs_init()
4696 qv->sw_thread_id) in mvpp2_irqs_init()
4697 cpumask_set_cpu(cpu, qv->mask); in mvpp2_irqs_init()
4700 irq_set_affinity_hint(qv->irq, qv->mask); in mvpp2_irqs_init()
4706 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_init()
4707 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_init()
4709 irq_set_affinity_hint(qv->irq, NULL); in mvpp2_irqs_init()
4710 kfree(qv->mask); in mvpp2_irqs_init()
4711 qv->mask = NULL; in mvpp2_irqs_init()
4712 free_irq(qv->irq, qv); in mvpp2_irqs_init()
4722 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_deinit()
4723 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_deinit()
4725 irq_set_affinity_hint(qv->irq, NULL); in mvpp2_irqs_deinit()
4726 kfree(qv->mask); in mvpp2_irqs_deinit()
4727 qv->mask = NULL; in mvpp2_irqs_deinit()
4728 irq_clear_status_flags(qv->irq, IRQ_NO_BALANCING); in mvpp2_irqs_deinit()
4729 free_irq(qv->irq, qv); in mvpp2_irqs_deinit()
4736 !(port->flags & MVPP2_F_LOOPBACK); in mvpp22_rss_is_supported()
4742 struct mvpp2 *priv = port->priv; in mvpp2_open()
4753 err = mvpp2_prs_mac_da_accept(port, dev->dev_addr, true); in mvpp2_open()
4758 err = mvpp2_prs_tag_mode_set(port->priv, port->id, MVPP2_TAG_TYPE_MH); in mvpp2_open()
4772 netdev_err(port->dev, "cannot allocate Rx queues\n"); in mvpp2_open()
4778 netdev_err(port->dev, "cannot allocate Tx queues\n"); in mvpp2_open()
4784 netdev_err(port->dev, "cannot init IRQs\n"); in mvpp2_open()
4788 if (port->phylink) { in mvpp2_open()
4789 err = phylink_fwnode_phy_connect(port->phylink, port->fwnode, 0); in mvpp2_open()
4791 netdev_err(port->dev, "could not attach PHY (%d)\n", in mvpp2_open()
4799 if (priv->hw_version >= MVPP22 && port->port_irq) { in mvpp2_open()
4800 err = request_irq(port->port_irq, mvpp2_port_isr, 0, in mvpp2_open()
4801 dev->name, port); in mvpp2_open()
4803 netdev_err(port->dev, in mvpp2_open()
4805 port->port_irq); in mvpp2_open()
4812 netif_carrier_off(port->dev); in mvpp2_open()
4816 port->port_irq = 0; in mvpp2_open()
4820 netdev_err(port->dev, in mvpp2_open()
4822 err = -ENOENT; in mvpp2_open()
4833 queue_delayed_work(priv->stats_queue, &port->stats_work, in mvpp2_open()
4859 if (port->phylink) in mvpp2_stop()
4860 phylink_disconnect_phy(port->phylink); in mvpp2_stop()
4861 if (port->port_irq) in mvpp2_stop()
4862 free_irq(port->port_irq, port); in mvpp2_stop()
4865 if (!port->has_tx_irqs) { in mvpp2_stop()
4866 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_stop()
4867 port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_stop()
4869 hrtimer_cancel(&port_pcpu->tx_done_timer); in mvpp2_stop()
4870 port_pcpu->timer_scheduled = false; in mvpp2_stop()
4876 cancel_delayed_work_sync(&port->stats_work); in mvpp2_stop()
4891 ret = mvpp2_prs_mac_da_accept(port, ha->addr, true); in mvpp2_prs_mac_da_accept_list()
4901 if (!enable && (port->dev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) in mvpp2_set_rx_promisc()
4906 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_promisc()
4909 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_promisc()
4920 if (dev->flags & IFF_PROMISC) { in mvpp2_set_rx_mode()
4928 mvpp2_prs_mac_da_accept_list(port, &dev->uc)) in mvpp2_set_rx_mode()
4929 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4932 if (dev->flags & IFF_ALLMULTI) { in mvpp2_set_rx_mode()
4933 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4939 mvpp2_prs_mac_da_accept_list(port, &dev->mc)) in mvpp2_set_rx_mode()
4940 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4949 if (!is_valid_ether_addr(addr->sa_data)) in mvpp2_set_mac_address()
4950 return -EADDRNOTAVAIL; in mvpp2_set_mac_address()
4952 err = mvpp2_prs_update_mac_da(dev, addr->sa_data); in mvpp2_set_mac_address()
4955 mvpp2_prs_update_mac_da(dev, dev->dev_addr); in mvpp2_set_mac_address()
4966 bool change_percpu = (percpu != priv->percpu_pools); in mvpp2_bm_switch_buffers()
4971 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_switch_buffers()
4972 port = priv->port_list[i]; in mvpp2_bm_switch_buffers()
4973 status[i] = netif_running(port->dev); in mvpp2_bm_switch_buffers()
4975 mvpp2_stop(port->dev); in mvpp2_bm_switch_buffers()
4979 if (priv->percpu_pools) in mvpp2_bm_switch_buffers()
4980 numbufs = port->nrxqs * 2; in mvpp2_bm_switch_buffers()
4986 mvpp2_bm_pool_destroy(port->dev->dev.parent, priv, &priv->bm_pools[i]); in mvpp2_bm_switch_buffers()
4988 devm_kfree(port->dev->dev.parent, priv->bm_pools); in mvpp2_bm_switch_buffers()
4989 priv->percpu_pools = percpu; in mvpp2_bm_switch_buffers()
4990 mvpp2_bm_init(port->dev->dev.parent, priv); in mvpp2_bm_switch_buffers()
4992 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_switch_buffers()
4993 port = priv->port_list[i]; in mvpp2_bm_switch_buffers()
4996 mvpp2_open(port->dev); in mvpp2_bm_switch_buffers()
5009 struct mvpp2 *priv = port->priv; in mvpp2_change_mtu()
5019 if (port->xdp_prog) { in mvpp2_change_mtu()
5021 return -EINVAL; in mvpp2_change_mtu()
5023 if (priv->percpu_pools) { in mvpp2_change_mtu()
5031 for (i = 0; i < priv->port_count; i++) in mvpp2_change_mtu()
5032 if (priv->port_list[i] != port && in mvpp2_change_mtu()
5033 MVPP2_RX_PKT_SIZE(priv->port_list[i]->dev->mtu) > in mvpp2_change_mtu()
5041 dev_info(port->dev->dev.parent, in mvpp2_change_mtu()
5042 "all ports have a low MTU, switching to per-cpu buffers"); in mvpp2_change_mtu()
5054 mvpp2_bm_update_mtu(dev, dev->mtu); in mvpp2_change_mtu()
5056 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu); in mvpp2_change_mtu()
5071 struct mvpp2 *priv = port->priv; in mvpp2_check_pagepool_dma()
5072 int err = -1, i; in mvpp2_check_pagepool_dma()
5074 if (!priv->percpu_pools) in mvpp2_check_pagepool_dma()
5077 if (!priv->page_pool[0]) in mvpp2_check_pagepool_dma()
5078 return -ENOMEM; in mvpp2_check_pagepool_dma()
5080 for (i = 0; i < priv->port_count; i++) { in mvpp2_check_pagepool_dma()
5081 port = priv->port_list[i]; in mvpp2_check_pagepool_dma()
5082 if (port->xdp_prog) { in mvpp2_check_pagepool_dma()
5089 if (priv->page_pool[0]->p.dma_dir != dma_dir) in mvpp2_check_pagepool_dma()
5109 cpu_stats = per_cpu_ptr(port->stats, cpu); in mvpp2_get_stats64()
5111 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); in mvpp2_get_stats64()
5112 rx_packets = cpu_stats->rx_packets; in mvpp2_get_stats64()
5113 rx_bytes = cpu_stats->rx_bytes; in mvpp2_get_stats64()
5114 tx_packets = cpu_stats->tx_packets; in mvpp2_get_stats64()
5115 tx_bytes = cpu_stats->tx_bytes; in mvpp2_get_stats64()
5116 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); in mvpp2_get_stats64()
5118 stats->rx_packets += rx_packets; in mvpp2_get_stats64()
5119 stats->rx_bytes += rx_bytes; in mvpp2_get_stats64()
5120 stats->tx_packets += tx_packets; in mvpp2_get_stats64()
5121 stats->tx_bytes += tx_bytes; in mvpp2_get_stats64()
5124 stats->rx_errors = dev->stats.rx_errors; in mvpp2_get_stats64()
5125 stats->rx_dropped = dev->stats.rx_dropped; in mvpp2_get_stats64()
5126 stats->tx_dropped = dev->stats.tx_dropped; in mvpp2_get_stats64()
5135 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) in mvpp2_set_ts_config()
5136 return -EFAULT; in mvpp2_set_ts_config()
5139 return -EINVAL; in mvpp2_set_ts_config()
5143 return -ERANGE; in mvpp2_set_ts_config()
5145 ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_set_ts_config()
5160 mvpp22_tai_start(port->priv->tai); in mvpp2_set_ts_config()
5168 port->rx_hwtstamp = true; in mvpp2_set_ts_config()
5170 port->rx_hwtstamp = false; in mvpp2_set_ts_config()
5182 mvpp22_tai_stop(port->priv->tai); in mvpp2_set_ts_config()
5184 port->tx_hwtstamp_type = config.tx_type; in mvpp2_set_ts_config()
5186 if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) in mvpp2_set_ts_config()
5187 return -EFAULT; in mvpp2_set_ts_config()
5198 config.tx_type = port->tx_hwtstamp_type; in mvpp2_get_ts_config()
5199 config.rx_filter = port->rx_hwtstamp ? in mvpp2_get_ts_config()
5202 if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) in mvpp2_get_ts_config()
5203 return -EFAULT; in mvpp2_get_ts_config()
5213 if (!port->hwtstamp) in mvpp2_ethtool_get_ts_info()
5214 return -EOPNOTSUPP; in mvpp2_ethtool_get_ts_info()
5216 info->phc_index = mvpp22_tai_ptp_clock_index(port->priv->tai); in mvpp2_ethtool_get_ts_info()
5217 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | in mvpp2_ethtool_get_ts_info()
5223 info->tx_types = BIT(HWTSTAMP_TX_OFF) | in mvpp2_ethtool_get_ts_info()
5225 info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | in mvpp2_ethtool_get_ts_info()
5237 if (port->hwtstamp) in mvpp2_ioctl()
5242 if (port->hwtstamp) in mvpp2_ioctl()
5247 if (!port->phylink) in mvpp2_ioctl()
5248 return -ENOTSUPP; in mvpp2_ioctl()
5250 return phylink_mii_ioctl(port->phylink, ifr, cmd); in mvpp2_ioctl()
5260 netdev_err(dev, "rx-vlan-filter offloading cannot accept more than %d VIDs per port\n", in mvpp2_vlan_rx_add_vid()
5261 MVPP2_PRS_VLAN_FILT_MAX - 1); in mvpp2_vlan_rx_add_vid()
5276 netdev_features_t changed = dev->features ^ features; in mvpp2_set_features()
5304 struct bpf_prog *prog = bpf->prog, *old_prog; in mvpp2_xdp_setup()
5305 bool running = netif_running(port->dev); in mvpp2_xdp_setup()
5306 bool reset = !prog != !port->xdp_prog; in mvpp2_xdp_setup()
5308 if (port->dev->mtu > ETH_DATA_LEN) { in mvpp2_xdp_setup()
5309 NL_SET_ERR_MSG_MOD(bpf->extack, "XDP is not supported with jumbo frames enabled"); in mvpp2_xdp_setup()
5310 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5313 if (!port->priv->percpu_pools) { in mvpp2_xdp_setup()
5314 NL_SET_ERR_MSG_MOD(bpf->extack, "Per CPU Pools required for XDP"); in mvpp2_xdp_setup()
5315 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5318 if (port->ntxqs < num_possible_cpus() * 2) { in mvpp2_xdp_setup()
5319 NL_SET_ERR_MSG_MOD(bpf->extack, "XDP_TX needs two TX queues per CPU"); in mvpp2_xdp_setup()
5320 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5325 mvpp2_stop(port->dev); in mvpp2_xdp_setup()
5327 old_prog = xchg(&port->xdp_prog, prog); in mvpp2_xdp_setup()
5337 mvpp2_open(port->dev); in mvpp2_xdp_setup()
5349 switch (xdp->command) { in mvpp2_xdp()
5353 return -EINVAL; in mvpp2_xdp()
5363 if (!port->phylink) in mvpp2_ethtool_nway_reset()
5364 return -ENOTSUPP; in mvpp2_ethtool_nway_reset()
5366 return phylink_ethtool_nway_reset(port->phylink); in mvpp2_ethtool_nway_reset()
5379 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_ethtool_set_coalesce()
5380 struct mvpp2_rx_queue *rxq = port->rxqs[queue]; in mvpp2_ethtool_set_coalesce()
5382 rxq->time_coal = c->rx_coalesce_usecs; in mvpp2_ethtool_set_coalesce()
5383 rxq->pkts_coal = c->rx_max_coalesced_frames; in mvpp2_ethtool_set_coalesce()
5388 if (port->has_tx_irqs) { in mvpp2_ethtool_set_coalesce()
5389 port->tx_time_coal = c->tx_coalesce_usecs; in mvpp2_ethtool_set_coalesce()
5393 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_ethtool_set_coalesce()
5394 struct mvpp2_tx_queue *txq = port->txqs[queue]; in mvpp2_ethtool_set_coalesce()
5396 txq->done_pkts_coal = c->tx_max_coalesced_frames; in mvpp2_ethtool_set_coalesce()
5398 if (port->has_tx_irqs) in mvpp2_ethtool_set_coalesce()
5414 c->rx_coalesce_usecs = port->rxqs[0]->time_coal; in mvpp2_ethtool_get_coalesce()
5415 c->rx_max_coalesced_frames = port->rxqs[0]->pkts_coal; in mvpp2_ethtool_get_coalesce()
5416 c->tx_max_coalesced_frames = port->txqs[0]->done_pkts_coal; in mvpp2_ethtool_get_coalesce()
5417 c->tx_coalesce_usecs = port->tx_time_coal; in mvpp2_ethtool_get_coalesce()
5424 strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME, in mvpp2_ethtool_get_drvinfo()
5425 sizeof(drvinfo->driver)); in mvpp2_ethtool_get_drvinfo()
5426 strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION, in mvpp2_ethtool_get_drvinfo()
5427 sizeof(drvinfo->version)); in mvpp2_ethtool_get_drvinfo()
5428 strlcpy(drvinfo->bus_info, dev_name(&dev->dev), in mvpp2_ethtool_get_drvinfo()
5429 sizeof(drvinfo->bus_info)); in mvpp2_ethtool_get_drvinfo()
5437 ring->rx_max_pending = MVPP2_MAX_RXD_MAX; in mvpp2_ethtool_get_ringparam()
5438 ring->tx_max_pending = MVPP2_MAX_TXD_MAX; in mvpp2_ethtool_get_ringparam()
5439 ring->rx_pending = port->rx_ring_size; in mvpp2_ethtool_get_ringparam()
5440 ring->tx_pending = port->tx_ring_size; in mvpp2_ethtool_get_ringparam()
5447 u16 prev_rx_ring_size = port->rx_ring_size; in mvpp2_ethtool_set_ringparam()
5448 u16 prev_tx_ring_size = port->tx_ring_size; in mvpp2_ethtool_set_ringparam()
5456 port->rx_ring_size = ring->rx_pending; in mvpp2_ethtool_set_ringparam()
5457 port->tx_ring_size = ring->tx_pending; in mvpp2_ethtool_set_ringparam()
5468 port->rx_ring_size = ring->rx_pending; in mvpp2_ethtool_set_ringparam()
5469 port->tx_ring_size = ring->tx_pending; in mvpp2_ethtool_set_ringparam()
5474 port->rx_ring_size = prev_rx_ring_size; in mvpp2_ethtool_set_ringparam()
5475 ring->rx_pending = prev_rx_ring_size; in mvpp2_ethtool_set_ringparam()
5483 port->tx_ring_size = prev_tx_ring_size; in mvpp2_ethtool_set_ringparam()
5484 ring->tx_pending = prev_tx_ring_size; in mvpp2_ethtool_set_ringparam()
5508 if (!port->phylink) in mvpp2_ethtool_get_pause_param()
5511 phylink_ethtool_get_pauseparam(port->phylink, pause); in mvpp2_ethtool_get_pause_param()
5519 if (!port->phylink) in mvpp2_ethtool_set_pause_param()
5520 return -ENOTSUPP; in mvpp2_ethtool_set_pause_param()
5522 return phylink_ethtool_set_pauseparam(port->phylink, pause); in mvpp2_ethtool_set_pause_param()
5530 if (!port->phylink) in mvpp2_ethtool_get_link_ksettings()
5531 return -ENOTSUPP; in mvpp2_ethtool_get_link_ksettings()
5533 return phylink_ethtool_ksettings_get(port->phylink, cmd); in mvpp2_ethtool_get_link_ksettings()
5541 if (!port->phylink) in mvpp2_ethtool_set_link_ksettings()
5542 return -ENOTSUPP; in mvpp2_ethtool_set_link_ksettings()
5544 return phylink_ethtool_ksettings_set(port->phylink, cmd); in mvpp2_ethtool_set_link_ksettings()
5554 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxnfc()
5556 switch (info->cmd) { in mvpp2_ethtool_get_rxnfc()
5561 info->data = port->nrxqs; in mvpp2_ethtool_get_rxnfc()
5564 info->rule_cnt = port->n_rfs_rules; in mvpp2_ethtool_get_rxnfc()
5571 if (port->rfs_rules[i]) in mvpp2_ethtool_get_rxnfc()
5576 return -ENOTSUPP; in mvpp2_ethtool_get_rxnfc()
5589 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxnfc()
5591 switch (info->cmd) { in mvpp2_ethtool_set_rxnfc()
5602 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxnfc()
5621 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxfh()
5639 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5642 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5645 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5660 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxfh_context()
5662 return -EINVAL; in mvpp2_ethtool_get_rxfh_context()
5682 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5685 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5688 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5747 * had a single IRQ defined per-port.
5752 struct mvpp2_queue_vector *v = &port->qvecs[0]; in mvpp2_simple_queue_vectors_init()
5754 v->first_rxq = 0; in mvpp2_simple_queue_vectors_init()
5755 v->nrxqs = port->nrxqs; in mvpp2_simple_queue_vectors_init()
5756 v->type = MVPP2_QUEUE_VECTOR_SHARED; in mvpp2_simple_queue_vectors_init()
5757 v->sw_thread_id = 0; in mvpp2_simple_queue_vectors_init()
5758 v->sw_thread_mask = *cpumask_bits(cpu_online_mask); in mvpp2_simple_queue_vectors_init()
5759 v->port = port; in mvpp2_simple_queue_vectors_init()
5760 v->irq = irq_of_parse_and_map(port_node, 0); in mvpp2_simple_queue_vectors_init()
5761 if (v->irq <= 0) in mvpp2_simple_queue_vectors_init()
5762 return -EINVAL; in mvpp2_simple_queue_vectors_init()
5763 netif_napi_add(port->dev, &v->napi, mvpp2_poll, in mvpp2_simple_queue_vectors_init()
5766 port->nqvecs = 1; in mvpp2_simple_queue_vectors_init()
5774 struct mvpp2 *priv = port->priv; in mvpp2_multi_queue_vectors_init()
5780 port->nqvecs = priv->nthreads + 1; in mvpp2_multi_queue_vectors_init()
5783 port->nqvecs = priv->nthreads; in mvpp2_multi_queue_vectors_init()
5787 for (i = 0; i < port->nqvecs; i++) { in mvpp2_multi_queue_vectors_init()
5790 v = port->qvecs + i; in mvpp2_multi_queue_vectors_init()
5792 v->port = port; in mvpp2_multi_queue_vectors_init()
5793 v->type = MVPP2_QUEUE_VECTOR_PRIVATE; in mvpp2_multi_queue_vectors_init()
5794 v->sw_thread_id = i; in mvpp2_multi_queue_vectors_init()
5795 v->sw_thread_mask = BIT(i); in mvpp2_multi_queue_vectors_init()
5797 if (port->flags & MVPP2_F_DT_COMPAT) in mvpp2_multi_queue_vectors_init()
5798 snprintf(irqname, sizeof(irqname), "tx-cpu%d", i); in mvpp2_multi_queue_vectors_init()
5803 v->first_rxq = i; in mvpp2_multi_queue_vectors_init()
5804 v->nrxqs = 1; in mvpp2_multi_queue_vectors_init()
5806 i == (port->nqvecs - 1)) { in mvpp2_multi_queue_vectors_init()
5807 v->first_rxq = 0; in mvpp2_multi_queue_vectors_init()
5808 v->nrxqs = port->nrxqs; in mvpp2_multi_queue_vectors_init()
5809 v->type = MVPP2_QUEUE_VECTOR_SHARED; in mvpp2_multi_queue_vectors_init()
5811 if (port->flags & MVPP2_F_DT_COMPAT) in mvpp2_multi_queue_vectors_init()
5812 strncpy(irqname, "rx-shared", sizeof(irqname)); in mvpp2_multi_queue_vectors_init()
5816 v->irq = of_irq_get_byname(port_node, irqname); in mvpp2_multi_queue_vectors_init()
5818 v->irq = fwnode_irq_get(port->fwnode, i); in mvpp2_multi_queue_vectors_init()
5819 if (v->irq <= 0) { in mvpp2_multi_queue_vectors_init()
5820 ret = -EINVAL; in mvpp2_multi_queue_vectors_init()
5824 netif_napi_add(port->dev, &v->napi, mvpp2_poll, in mvpp2_multi_queue_vectors_init()
5831 for (i = 0; i < port->nqvecs; i++) in mvpp2_multi_queue_vectors_init()
5832 irq_dispose_mapping(port->qvecs[i].irq); in mvpp2_multi_queue_vectors_init()
5839 if (port->has_tx_irqs) in mvpp2_queue_vectors_init()
5849 for (i = 0; i < port->nqvecs; i++) in mvpp2_queue_vectors_deinit()
5850 irq_dispose_mapping(port->qvecs[i].irq); in mvpp2_queue_vectors_deinit()
5856 struct mvpp2 *priv = port->priv; in mvpp2_rx_irqs_setup()
5860 if (priv->hw_version == MVPP21) { in mvpp2_rx_irqs_setup()
5861 mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id), in mvpp2_rx_irqs_setup()
5862 port->nrxqs); in mvpp2_rx_irqs_setup()
5867 for (i = 0; i < port->nqvecs; i++) { in mvpp2_rx_irqs_setup()
5868 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_rx_irqs_setup()
5870 if (!qv->nrxqs) in mvpp2_rx_irqs_setup()
5873 val = qv->sw_thread_id; in mvpp2_rx_irqs_setup()
5874 val |= port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET; in mvpp2_rx_irqs_setup()
5877 val = qv->first_rxq; in mvpp2_rx_irqs_setup()
5878 val |= qv->nrxqs << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET; in mvpp2_rx_irqs_setup()
5886 struct device *dev = port->dev->dev.parent; in mvpp2_port_init()
5887 struct mvpp2 *priv = port->priv; in mvpp2_port_init()
5893 if (port->first_rxq + port->nrxqs > in mvpp2_port_init()
5894 MVPP2_MAX_PORTS * priv->max_port_rxqs) in mvpp2_port_init()
5895 return -EINVAL; in mvpp2_port_init()
5897 if (port->nrxqs > priv->max_port_rxqs || port->ntxqs > MVPP2_MAX_TXQ) in mvpp2_port_init()
5898 return -EINVAL; in mvpp2_port_init()
5904 if (mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_init()
5905 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_init()
5908 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_init()
5910 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_port_init()
5913 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_port_init()
5916 port->tx_time_coal = MVPP2_TXDONE_COAL_USEC; in mvpp2_port_init()
5918 port->txqs = devm_kcalloc(dev, port->ntxqs, sizeof(*port->txqs), in mvpp2_port_init()
5920 if (!port->txqs) in mvpp2_port_init()
5921 return -ENOMEM; in mvpp2_port_init()
5926 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_port_init()
5927 int queue_phy_id = mvpp2_txq_phys(port->id, queue); in mvpp2_port_init()
5932 err = -ENOMEM; in mvpp2_port_init()
5936 txq->pcpu = alloc_percpu(struct mvpp2_txq_pcpu); in mvpp2_port_init()
5937 if (!txq->pcpu) { in mvpp2_port_init()
5938 err = -ENOMEM; in mvpp2_port_init()
5942 txq->id = queue_phy_id; in mvpp2_port_init()
5943 txq->log_id = queue; in mvpp2_port_init()
5944 txq->done_pkts_coal = MVPP2_TXDONE_COAL_PKTS_THRESH; in mvpp2_port_init()
5945 for (thread = 0; thread < priv->nthreads; thread++) { in mvpp2_port_init()
5946 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_port_init()
5947 txq_pcpu->thread = thread; in mvpp2_port_init()
5950 port->txqs[queue] = txq; in mvpp2_port_init()
5953 port->rxqs = devm_kcalloc(dev, port->nrxqs, sizeof(*port->rxqs), in mvpp2_port_init()
5955 if (!port->rxqs) { in mvpp2_port_init()
5956 err = -ENOMEM; in mvpp2_port_init()
5961 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_port_init()
5967 err = -ENOMEM; in mvpp2_port_init()
5971 rxq->id = port->first_rxq + queue; in mvpp2_port_init()
5972 rxq->port = port->id; in mvpp2_port_init()
5973 rxq->logic_rxq = queue; in mvpp2_port_init()
5975 port->rxqs[queue] = rxq; in mvpp2_port_init()
5981 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_port_init()
5982 struct mvpp2_rx_queue *rxq = port->rxqs[queue]; in mvpp2_port_init()
5984 rxq->size = port->rx_ring_size; in mvpp2_port_init()
5985 rxq->pkts_coal = MVPP2_RX_COAL_PKTS; in mvpp2_port_init()
5986 rxq->time_coal = MVPP2_RX_COAL_USEC; in mvpp2_port_init()
6002 port->pkt_size = MVPP2_RX_PKT_SIZE(port->dev->mtu); in mvpp2_port_init()
6011 memset(port->ethtool_stats, 0, in mvpp2_port_init()
6012 MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs) * sizeof(u64)); in mvpp2_port_init()
6017 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_port_init()
6018 if (!port->txqs[queue]) in mvpp2_port_init()
6020 free_percpu(port->txqs[queue]->pcpu); in mvpp2_port_init()
6028 char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1", "tx-cpu2", in mvpp22_port_has_legacy_tx_irqs()
6029 "tx-cpu3" }; in mvpp22_port_has_legacy_tx_irqs()
6033 if (of_property_match_string(port_node, "interrupt-names", in mvpp22_port_has_legacy_tx_irqs()
6042 * - PPv2.1: there are no such interrupts.
6043 * - PPv2.2 and PPv2.3:
6044 * - The old DTs have: "rx-shared", "tx-cpuX" with X in [0...3]
6045 * - The new ones have: "hifX" with X in [0..8]
6060 if (priv->hw_version == MVPP21) in mvpp2_port_has_irqs()
6068 if (of_property_match_string(port_node, "interrupt-names", in mvpp2_port_has_irqs()
6086 ether_addr_copy(dev->dev_addr, fw_mac_addr); in mvpp2_port_copy_mac_addr()
6090 if (priv->hw_version == MVPP21) { in mvpp2_port_copy_mac_addr()
6094 ether_addr_copy(dev->dev_addr, hw_mac_addr); in mvpp2_port_copy_mac_addr()
6119 state->speed = SPEED_10000; in mvpp2_xlg_pcs_get_state()
6120 state->duplex = 1; in mvpp2_xlg_pcs_get_state()
6121 state->an_complete = 1; in mvpp2_xlg_pcs_get_state()
6123 val = readl(port->base + MVPP22_XLG_STATUS); in mvpp2_xlg_pcs_get_state()
6124 state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP); in mvpp2_xlg_pcs_get_state()
6126 state->pause = 0; in mvpp2_xlg_pcs_get_state()
6127 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_xlg_pcs_get_state()
6129 state->pause |= MLO_PAUSE_TX; in mvpp2_xlg_pcs_get_state()
6131 state->pause |= MLO_PAUSE_RX; in mvpp2_xlg_pcs_get_state()
6154 val = readl(port->base + MVPP2_GMAC_STATUS0); in mvpp2_gmac_pcs_get_state()
6156 state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE); in mvpp2_gmac_pcs_get_state()
6157 state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP); in mvpp2_gmac_pcs_get_state()
6158 state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX); in mvpp2_gmac_pcs_get_state()
6160 switch (port->phy_interface) { in mvpp2_gmac_pcs_get_state()
6162 state->speed = SPEED_1000; in mvpp2_gmac_pcs_get_state()
6165 state->speed = SPEED_2500; in mvpp2_gmac_pcs_get_state()
6169 state->speed = SPEED_1000; in mvpp2_gmac_pcs_get_state()
6171 state->speed = SPEED_100; in mvpp2_gmac_pcs_get_state()
6173 state->speed = SPEED_10; in mvpp2_gmac_pcs_get_state()
6176 state->pause = 0; in mvpp2_gmac_pcs_get_state()
6178 state->pause |= MLO_PAUSE_RX; in mvpp2_gmac_pcs_get_state()
6180 state->pause |= MLO_PAUSE_TX; in mvpp2_gmac_pcs_get_state()
6230 old_an = an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_config()
6234 writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_config()
6243 u32 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6246 port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6248 port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6265 switch (state->interface) { in mvpp2_phylink_validate()
6281 * Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... in mvpp2_phylink_validate()
6282 * When <PortType> = 1 (1000BASE-X) this field must be set to 1. in mvpp2_phylink_validate()
6284 if (!phylink_test(state->advertising, Autoneg)) in mvpp2_phylink_validate()
6294 if (port->priv->global_tx_fc) { in mvpp2_phylink_validate()
6299 switch (state->interface) { in mvpp2_phylink_validate()
6312 if (state->interface != PHY_INTERFACE_MODE_NA) in mvpp2_phylink_validate()
6320 phylink_set(mask, 10baseT_Half); in mvpp2_phylink_validate()
6321 phylink_set(mask, 10baseT_Full); in mvpp2_phylink_validate()
6326 if (state->interface != PHY_INTERFACE_MODE_NA) in mvpp2_phylink_validate()
6331 if (port->comphy || in mvpp2_phylink_validate()
6332 state->interface != PHY_INTERFACE_MODE_2500BASEX) { in mvpp2_phylink_validate()
6336 if (port->comphy || in mvpp2_phylink_validate()
6337 state->interface == PHY_INTERFACE_MODE_2500BASEX) { in mvpp2_phylink_validate()
6347 bitmap_and(state->advertising, state->advertising, mask, in mvpp2_phylink_validate()
6362 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_xlg_config()
6365 mvpp2_modify(port->base + MVPP22_XLG_CTRL4_REG, in mvpp2_xlg_config()
6373 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_xlg_config()
6384 old_ctrl0 = ctrl0 = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_config()
6385 old_ctrl2 = ctrl2 = readl(port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_gmac_config()
6386 old_ctrl4 = ctrl4 = readl(port->base + MVPP22_GMAC_CTRL_4_REG); in mvpp2_gmac_config()
6392 if (phy_interface_mode_is_8023z(state->interface)) { in mvpp2_gmac_config()
6398 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_gmac_config()
6404 } else if (phy_interface_mode_is_rgmii(state->interface)) { in mvpp2_gmac_config()
6413 /* Phy or fixed speed - no in-band AN, nothing to do, leave the in mvpp2_gmac_config()
6414 * configured speed, duplex and flow control as-is. in mvpp2_gmac_config()
6416 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_gmac_config()
6417 /* SGMII in-band mode receives the speed and duplex from in mvpp2_gmac_config()
6419 } else if (phy_interface_mode_is_8023z(state->interface)) { in mvpp2_gmac_config()
6429 writel(ctrl0, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_config()
6431 writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_gmac_config()
6433 writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG); in mvpp2_gmac_config()
6442 if (mvpp2_is_xlg(interface) && port->gop_id != 0) { in mvpp2__mac_prepare()
6443 netdev_err(port->dev, "Invalid mode on %s\n", port->dev->name); in mvpp2__mac_prepare()
6444 return -EINVAL; in mvpp2__mac_prepare()
6447 if (port->phy_interface != interface || in mvpp2__mac_prepare()
6450 * in-band mode to ensure we do not change the configuration in mvpp2__mac_prepare()
6455 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2__mac_prepare()
6461 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2__mac_prepare()
6470 if (port->phy_interface != interface) { in mvpp2__mac_prepare()
6472 mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG, in mvpp2__mac_prepare()
6476 if (port->priv->hw_version >= MVPP22) { in mvpp2__mac_prepare()
6479 phy_power_off(port->comphy); in mvpp2__mac_prepare()
6488 port->phylink_pcs.ops = &mvpp2_phylink_xlg_pcs_ops; in mvpp2__mac_prepare()
6490 port->phylink_pcs.ops = &mvpp2_phylink_gmac_pcs_ops; in mvpp2__mac_prepare()
6503 phylink_set_pcs(port->phylink, &port->phylink_pcs); in mvpp2_mac_prepare()
6514 if (mvpp2_is_xlg(state->interface)) in mvpp2_mac_config()
6516 else if (phy_interface_mode_is_rgmii(state->interface) || in mvpp2_mac_config()
6517 phy_interface_mode_is_8023z(state->interface) || in mvpp2_mac_config()
6518 state->interface == PHY_INTERFACE_MODE_SGMII) in mvpp2_mac_config()
6521 if (port->priv->hw_version == MVPP21 && port->flags & MVPP2_F_LOOPBACK) in mvpp2_mac_config()
6530 if (port->priv->hw_version >= MVPP22 && in mvpp2_mac_finish()
6531 port->phy_interface != interface) { in mvpp2_mac_finish()
6532 port->phy_interface = interface; in mvpp2_mac_finish()
6543 mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG, in mvpp2_mac_finish()
6546 while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) & in mvpp2_mac_finish()
6553 /* Allow the link to come up if in in-band mode, otherwise the in mvpp2_mac_finish()
6558 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_mac_finish()
6562 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2_mac_finish()
6588 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_mac_link_up()
6606 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2_mac_link_up()
6624 mvpp2_modify(port->base + MVPP22_GMAC_CTRL_4_REG, in mvpp2_mac_link_up()
6629 if (port->priv->global_tx_fc) { in mvpp2_mac_link_up()
6630 port->tx_fc = tx_pause; in mvpp2_mac_link_up()
6635 if (port->priv->percpu_pools) { in mvpp2_mac_link_up()
6636 for (i = 0; i < port->nrxqs; i++) in mvpp2_mac_link_up()
6637 mvpp2_bm_pool_update_fc(port, &port->priv->bm_pools[i], tx_pause); in mvpp2_mac_link_up()
6639 mvpp2_bm_pool_update_fc(port, port->pool_long, tx_pause); in mvpp2_mac_link_up()
6640 mvpp2_bm_pool_update_fc(port, port->pool_short, tx_pause); in mvpp2_mac_link_up()
6642 if (port->priv->hw_version == MVPP23) in mvpp2_mac_link_up()
6643 mvpp23_rx_fifo_fc_en(port->priv, port->id, tx_pause); in mvpp2_mac_link_up()
6650 netif_tx_wake_all_queues(port->dev); in mvpp2_mac_link_up()
6661 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_link_down()
6664 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_link_down()
6666 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_mac_link_down()
6669 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_mac_link_down()
6673 netif_tx_stop_all_queues(port->dev); in mvpp2_mac_link_down()
6689 /* Work-around for ACPI */
6697 .interface = port->phy_interface, in mvpp2_acpi_start()
6699 mvpp2__mac_prepare(&port->phylink_config, MLO_AN_INBAND, in mvpp2_acpi_start()
6700 port->phy_interface); in mvpp2_acpi_start()
6701 mvpp2_mac_config(&port->phylink_config, MLO_AN_INBAND, &state); in mvpp2_acpi_start()
6702 port->phylink_pcs.ops->pcs_config(&port->phylink_pcs, MLO_AN_INBAND, in mvpp2_acpi_start()
6703 port->phy_interface, in mvpp2_acpi_start()
6705 mvpp2_mac_finish(&port->phylink_config, MLO_AN_INBAND, in mvpp2_acpi_start()
6706 port->phy_interface); in mvpp2_acpi_start()
6707 mvpp2_mac_link_up(&port->phylink_config, NULL, in mvpp2_acpi_start()
6708 MLO_AN_INBAND, port->phy_interface, in mvpp2_acpi_start()
6720 return (!fwnode_property_present(port_fwnode, "phy-handle") && in mvpp2_use_acpi_compat_mode()
6722 !fwnode_get_named_child_node(port_fwnode, "fixed-link")); in mvpp2_use_acpi_compat_mode()
6747 dev_err(&pdev->dev, in mvpp2_port_probe()
6749 return -EINVAL; in mvpp2_port_probe()
6757 return -ENOMEM; in mvpp2_port_probe()
6761 dev_err(&pdev->dev, "incorrect phy mode\n"); in mvpp2_port_probe()
6767 * Rewrite 10GBASE-KR to 10GBASE-R for compatibility with existing DT. in mvpp2_port_probe()
6768 * Existing usage of 10GBASE-KR is not correct; no backplane in mvpp2_port_probe()
6770 * 10GBASE-KR. in mvpp2_port_probe()
6776 comphy = devm_of_phy_get(&pdev->dev, port_node, NULL); in mvpp2_port_probe()
6778 if (PTR_ERR(comphy) == -EPROBE_DEFER) { in mvpp2_port_probe()
6779 err = -EPROBE_DEFER; in mvpp2_port_probe()
6786 if (fwnode_property_read_u32(port_fwnode, "port-id", &id)) { in mvpp2_port_probe()
6787 err = -EINVAL; in mvpp2_port_probe()
6788 dev_err(&pdev->dev, "missing port-id value\n"); in mvpp2_port_probe()
6792 dev->tx_queue_len = MVPP2_MAX_TXD_MAX; in mvpp2_port_probe()
6793 dev->watchdog_timeo = 5 * HZ; in mvpp2_port_probe()
6794 dev->netdev_ops = &mvpp2_netdev_ops; in mvpp2_port_probe()
6795 dev->ethtool_ops = &mvpp2_eth_tool_ops; in mvpp2_port_probe()
6798 port->dev = dev; in mvpp2_port_probe()
6799 port->fwnode = port_fwnode; in mvpp2_port_probe()
6800 port->ntxqs = ntxqs; in mvpp2_port_probe()
6801 port->nrxqs = nrxqs; in mvpp2_port_probe()
6802 port->priv = priv; in mvpp2_port_probe()
6803 port->has_tx_irqs = has_tx_irqs; in mvpp2_port_probe()
6804 port->flags = flags; in mvpp2_port_probe()
6811 port->port_irq = of_irq_get_byname(port_node, "link"); in mvpp2_port_probe()
6813 port->port_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1); in mvpp2_port_probe()
6814 if (port->port_irq == -EPROBE_DEFER) { in mvpp2_port_probe()
6815 err = -EPROBE_DEFER; in mvpp2_port_probe()
6818 if (port->port_irq <= 0) in mvpp2_port_probe()
6820 port->port_irq = 0; in mvpp2_port_probe()
6823 port->flags |= MVPP2_F_LOOPBACK; in mvpp2_port_probe()
6825 port->id = id; in mvpp2_port_probe()
6826 if (priv->hw_version == MVPP21) in mvpp2_port_probe()
6827 port->first_rxq = port->id * port->nrxqs; in mvpp2_port_probe()
6829 port->first_rxq = port->id * priv->max_port_rxqs; in mvpp2_port_probe()
6831 port->of_node = port_node; in mvpp2_port_probe()
6832 port->phy_interface = phy_mode; in mvpp2_port_probe()
6833 port->comphy = comphy; in mvpp2_port_probe()
6835 if (priv->hw_version == MVPP21) { in mvpp2_port_probe()
6836 port->base = devm_platform_ioremap_resource(pdev, 2 + id); in mvpp2_port_probe()
6837 if (IS_ERR(port->base)) { in mvpp2_port_probe()
6838 err = PTR_ERR(port->base); in mvpp2_port_probe()
6842 port->stats_base = port->priv->lms_base + in mvpp2_port_probe()
6844 port->gop_id * MVPP21_MIB_COUNTERS_PORT_SZ; in mvpp2_port_probe()
6846 if (fwnode_property_read_u32(port_fwnode, "gop-port-id", in mvpp2_port_probe()
6847 &port->gop_id)) { in mvpp2_port_probe()
6848 err = -EINVAL; in mvpp2_port_probe()
6849 dev_err(&pdev->dev, "missing gop-port-id value\n"); in mvpp2_port_probe()
6853 port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id); in mvpp2_port_probe()
6854 port->stats_base = port->priv->iface_base + in mvpp2_port_probe()
6856 port->gop_id * MVPP22_MIB_COUNTERS_PORT_SZ; in mvpp2_port_probe()
6861 if (priv->tai) in mvpp2_port_probe()
6862 port->hwtstamp = true; in mvpp2_port_probe()
6865 /* Alloc per-cpu and ethtool stats */ in mvpp2_port_probe()
6866 port->stats = netdev_alloc_pcpu_stats(struct mvpp2_pcpu_stats); in mvpp2_port_probe()
6867 if (!port->stats) { in mvpp2_port_probe()
6868 err = -ENOMEM; in mvpp2_port_probe()
6872 port->ethtool_stats = devm_kcalloc(&pdev->dev, in mvpp2_port_probe()
6875 if (!port->ethtool_stats) { in mvpp2_port_probe()
6876 err = -ENOMEM; in mvpp2_port_probe()
6880 mutex_init(&port->gather_stats_lock); in mvpp2_port_probe()
6881 INIT_DELAYED_WORK(&port->stats_work, mvpp2_gather_hw_statistics); in mvpp2_port_probe()
6885 port->tx_ring_size = MVPP2_MAX_TXD_DFLT; in mvpp2_port_probe()
6886 port->rx_ring_size = MVPP2_MAX_RXD_DFLT; in mvpp2_port_probe()
6887 SET_NETDEV_DEV(dev, &pdev->dev); in mvpp2_port_probe()
6891 dev_err(&pdev->dev, "failed to init port %d\n", id); in mvpp2_port_probe()
6900 port->pcpu = alloc_percpu(struct mvpp2_port_pcpu); in mvpp2_port_probe()
6901 if (!port->pcpu) { in mvpp2_port_probe()
6902 err = -ENOMEM; in mvpp2_port_probe()
6906 if (!port->has_tx_irqs) { in mvpp2_port_probe()
6907 for (thread = 0; thread < priv->nthreads; thread++) { in mvpp2_port_probe()
6908 port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_port_probe()
6910 hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC, in mvpp2_port_probe()
6912 port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb; in mvpp2_port_probe()
6913 port_pcpu->timer_scheduled = false; in mvpp2_port_probe()
6914 port_pcpu->dev = dev; in mvpp2_port_probe()
6920 dev->features = features | NETIF_F_RXCSUM; in mvpp2_port_probe()
6921 dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO | in mvpp2_port_probe()
6925 dev->hw_features |= NETIF_F_RXHASH; in mvpp2_port_probe()
6926 dev->features |= NETIF_F_NTUPLE; in mvpp2_port_probe()
6929 if (!port->priv->percpu_pools) in mvpp2_port_probe()
6930 mvpp2_set_hw_csum(port, port->pool_long->id); in mvpp2_port_probe()
6932 dev->vlan_features |= features; in mvpp2_port_probe()
6933 dev->gso_max_segs = MVPP2_MAX_TSO_SEGS; in mvpp2_port_probe()
6934 dev->priv_flags |= IFF_UNICAST_FLT; in mvpp2_port_probe()
6936 /* MTU range: 68 - 9704 */ in mvpp2_port_probe()
6937 dev->min_mtu = ETH_MIN_MTU; in mvpp2_port_probe()
6938 /* 9704 == 9728 - 20 and rounding to 8 */ in mvpp2_port_probe()
6939 dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE; in mvpp2_port_probe()
6940 dev->dev.of_node = port_node; in mvpp2_port_probe()
6943 port->phylink_config.dev = &dev->dev; in mvpp2_port_probe()
6944 port->phylink_config.type = PHYLINK_NETDEV; in mvpp2_port_probe()
6946 phylink = phylink_create(&port->phylink_config, port_fwnode, in mvpp2_port_probe()
6952 port->phylink = phylink; in mvpp2_port_probe()
6954 dev_warn(&pdev->dev, "Use link irqs for port#%d. FW update required\n", port->id); in mvpp2_port_probe()
6955 port->phylink = NULL; in mvpp2_port_probe()
6958 /* Cycle the comphy to power it down, saving 270mW per port - in mvpp2_port_probe()
6962 if (port->comphy) { in mvpp2_port_probe()
6965 phy_power_off(port->comphy); in mvpp2_port_probe()
6970 dev_err(&pdev->dev, "failed to register netdev\n"); in mvpp2_port_probe()
6973 netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr); in mvpp2_port_probe()
6975 priv->port_list[priv->port_count++] = port; in mvpp2_port_probe()
6980 if (port->phylink) in mvpp2_port_probe()
6981 phylink_destroy(port->phylink); in mvpp2_port_probe()
6983 free_percpu(port->pcpu); in mvpp2_port_probe()
6985 for (i = 0; i < port->ntxqs; i++) in mvpp2_port_probe()
6986 free_percpu(port->txqs[i]->pcpu); in mvpp2_port_probe()
6988 free_percpu(port->stats); in mvpp2_port_probe()
6990 if (port->port_irq) in mvpp2_port_probe()
6991 irq_dispose_mapping(port->port_irq); in mvpp2_port_probe()
7004 unregister_netdev(port->dev); in mvpp2_port_remove()
7005 if (port->phylink) in mvpp2_port_remove()
7006 phylink_destroy(port->phylink); in mvpp2_port_remove()
7007 free_percpu(port->pcpu); in mvpp2_port_remove()
7008 free_percpu(port->stats); in mvpp2_port_remove()
7009 for (i = 0; i < port->ntxqs; i++) in mvpp2_port_remove()
7010 free_percpu(port->txqs[i]->pcpu); in mvpp2_port_remove()
7012 if (port->port_irq) in mvpp2_port_remove()
7013 irq_dispose_mapping(port->port_irq); in mvpp2_port_remove()
7014 free_netdev(port->dev); in mvpp2_port_remove()
7034 for (i = 0; i < dram->num_cs; i++) { in mvpp2_conf_mbus_windows()
7035 const struct mbus_dram_window *cs = dram->cs + i; in mvpp2_conf_mbus_windows()
7038 (cs->base & 0xffff0000) | (cs->mbus_attr << 8) | in mvpp2_conf_mbus_windows()
7039 dram->mbus_dram_target_id); in mvpp2_conf_mbus_windows()
7042 (cs->size - 1) & 0xffff0000); in mvpp2_conf_mbus_windows()
7078 * Guarantee minimum 32kB for 10G port and 8kB for port 1, capable of 2.5G
7091 port_map = priv->port_map & ~BIT(MVPP2_LOOPBACK_PORT_INDEX); in mvpp22_rx_fifo_init()
7113 size_remainder -= size; in mvpp22_rx_fifo_init()
7114 remaining_ports_count--; in mvpp22_rx_fifo_init()
7129 /* Port 0: maximum speed -10Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7131 * Port 1: maximum speed -5Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7133 * Port 2: maximum speed -1Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7138 for (port = 0; port < (MVPP2_MAX_PORTS - 1); port++) { in mvpp23_rx_fifo_fc_set_tresh()
7184 * The 10G interface should use 10kB (which is maximum possible size
7197 port_map = priv->port_map & ~BIT(MVPP2_LOOPBACK_PORT_INDEX); in mvpp22_tx_fifo_init()
7216 size_remainder -= size; in mvpp22_tx_fifo_init()
7217 remaining_ports_count--; in mvpp22_tx_fifo_init()
7289 if (priv->hw_version >= MVPP22) in mvpp2_init()
7293 if (priv->hw_version == MVPP21) { in mvpp2_init()
7294 val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG); in mvpp2_init()
7296 writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG); in mvpp2_init()
7298 val = readl(priv->iface_base + MVPP22_SMI_MISC_CFG_REG); in mvpp2_init()
7300 writel(val, priv->iface_base + MVPP22_SMI_MISC_CFG_REG); in mvpp2_init()
7304 priv->aggr_txqs = devm_kcalloc(&pdev->dev, MVPP2_MAX_THREADS, in mvpp2_init()
7305 sizeof(*priv->aggr_txqs), in mvpp2_init()
7307 if (!priv->aggr_txqs) in mvpp2_init()
7308 return -ENOMEM; in mvpp2_init()
7311 priv->aggr_txqs[i].id = i; in mvpp2_init()
7312 priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE; in mvpp2_init()
7313 err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv); in mvpp2_init()
7319 if (priv->hw_version == MVPP21) { in mvpp2_init()
7324 if (priv->hw_version == MVPP23) in mvpp2_init()
7328 if (priv->hw_version == MVPP21) in mvpp2_init()
7330 priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG); in mvpp2_init()
7336 err = mvpp2_bm_init(&pdev->dev, priv); in mvpp2_init()
7358 if (has_acpi_companion(&pdev->dev)) in mvpp2_get_sram()
7359 dev_warn(&pdev->dev, "ACPI is too old, Flow control not supported\n"); in mvpp2_get_sram()
7361 dev_warn(&pdev->dev, "DT is too old, Flow control not supported\n"); in mvpp2_get_sram()
7365 priv->cm3_base = devm_ioremap_resource(&pdev->dev, res); in mvpp2_get_sram()
7367 return PTR_ERR_OR_ZERO(priv->cm3_base); in mvpp2_get_sram()
7372 struct fwnode_handle *fwnode = pdev->dev.fwnode; in mvpp2_probe()
7380 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); in mvpp2_probe()
7382 return -ENOMEM; in mvpp2_probe()
7384 priv->hw_version = (unsigned long)device_get_match_data(&pdev->dev); in mvpp2_probe()
7389 if (priv->hw_version == MVPP21) in mvpp2_probe()
7396 if (priv->hw_version == MVPP21) { in mvpp2_probe()
7397 priv->lms_base = devm_platform_ioremap_resource(pdev, 1); in mvpp2_probe()
7398 if (IS_ERR(priv->lms_base)) in mvpp2_probe()
7399 return PTR_ERR(priv->lms_base); in mvpp2_probe()
7403 dev_err(&pdev->dev, "Invalid resource\n"); in mvpp2_probe()
7404 return -EINVAL; in mvpp2_probe()
7406 if (has_acpi_companion(&pdev->dev)) { in mvpp2_probe()
7408 * the ACPI, it can already appear as 'in-use' in mvpp2_probe()
7417 priv->iface_base = devm_ioremap_resource(&pdev->dev, res); in mvpp2_probe()
7418 if (IS_ERR(priv->iface_base)) in mvpp2_probe()
7419 return PTR_ERR(priv->iface_base); in mvpp2_probe()
7424 dev_warn(&pdev->dev, "Fail to alloc CM3 SRAM\n"); in mvpp2_probe()
7427 if (priv->cm3_base) in mvpp2_probe()
7428 priv->global_tx_fc = true; in mvpp2_probe()
7431 if (priv->hw_version >= MVPP22 && dev_of_node(&pdev->dev)) { in mvpp2_probe()
7432 priv->sysctrl_base = in mvpp2_probe()
7433 syscon_regmap_lookup_by_phandle(pdev->dev.of_node, in mvpp2_probe()
7434 "marvell,system-controller"); in mvpp2_probe()
7435 if (IS_ERR(priv->sysctrl_base)) in mvpp2_probe()
7441 priv->sysctrl_base = NULL; in mvpp2_probe()
7444 if (priv->hw_version >= MVPP22 && in mvpp2_probe()
7446 priv->percpu_pools = 1; in mvpp2_probe()
7451 priv->nthreads = min_t(unsigned int, num_present_cpus(), in mvpp2_probe()
7454 shared = num_present_cpus() - priv->nthreads; in mvpp2_probe()
7456 bitmap_fill(&priv->lock_map, in mvpp2_probe()
7462 addr_space_sz = (priv->hw_version == MVPP21 ? in mvpp2_probe()
7464 priv->swth_base[i] = base + i * addr_space_sz; in mvpp2_probe()
7467 if (priv->hw_version == MVPP21) in mvpp2_probe()
7468 priv->max_port_rxqs = 8; in mvpp2_probe()
7470 priv->max_port_rxqs = 32; in mvpp2_probe()
7472 if (dev_of_node(&pdev->dev)) { in mvpp2_probe()
7473 priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk"); in mvpp2_probe()
7474 if (IS_ERR(priv->pp_clk)) in mvpp2_probe()
7475 return PTR_ERR(priv->pp_clk); in mvpp2_probe()
7476 err = clk_prepare_enable(priv->pp_clk); in mvpp2_probe()
7480 priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk"); in mvpp2_probe()
7481 if (IS_ERR(priv->gop_clk)) { in mvpp2_probe()
7482 err = PTR_ERR(priv->gop_clk); in mvpp2_probe()
7485 err = clk_prepare_enable(priv->gop_clk); in mvpp2_probe()
7489 if (priv->hw_version >= MVPP22) { in mvpp2_probe()
7490 priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk"); in mvpp2_probe()
7491 if (IS_ERR(priv->mg_clk)) { in mvpp2_probe()
7492 err = PTR_ERR(priv->mg_clk); in mvpp2_probe()
7496 err = clk_prepare_enable(priv->mg_clk); in mvpp2_probe()
7500 priv->mg_core_clk = devm_clk_get_optional(&pdev->dev, "mg_core_clk"); in mvpp2_probe()
7501 if (IS_ERR(priv->mg_core_clk)) { in mvpp2_probe()
7502 err = PTR_ERR(priv->mg_core_clk); in mvpp2_probe()
7506 err = clk_prepare_enable(priv->mg_core_clk); in mvpp2_probe()
7511 priv->axi_clk = devm_clk_get_optional(&pdev->dev, "axi_clk"); in mvpp2_probe()
7512 if (IS_ERR(priv->axi_clk)) { in mvpp2_probe()
7513 err = PTR_ERR(priv->axi_clk); in mvpp2_probe()
7517 err = clk_prepare_enable(priv->axi_clk); in mvpp2_probe()
7522 priv->tclk = clk_get_rate(priv->pp_clk); in mvpp2_probe()
7524 err = device_property_read_u32(&pdev->dev, "clock-frequency", &priv->tclk); in mvpp2_probe()
7526 dev_err(&pdev->dev, "missing clock-frequency value\n"); in mvpp2_probe()
7531 if (priv->hw_version >= MVPP22) { in mvpp2_probe()
7532 err = dma_set_mask(&pdev->dev, MVPP2_DESC_DMA_MASK); in mvpp2_probe()
7540 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); in mvpp2_probe()
7545 /* Map DTS-active ports. Should be done before FIFO mvpp2_init */ in mvpp2_probe()
7547 if (!fwnode_property_read_u32(port_fwnode, "port-id", &i)) in mvpp2_probe()
7548 priv->port_map |= BIT(i); in mvpp2_probe()
7552 priv->hw_version = MVPP23; in mvpp2_probe()
7555 spin_lock_init(&priv->mss_spinlock); in mvpp2_probe()
7560 dev_err(&pdev->dev, "failed to initialize controller\n"); in mvpp2_probe()
7564 err = mvpp22_tai_probe(&pdev->dev, priv); in mvpp2_probe()
7575 if (priv->port_count == 0) { in mvpp2_probe()
7576 dev_err(&pdev->dev, "no ports enabled\n"); in mvpp2_probe()
7577 err = -ENODEV; in mvpp2_probe()
7582 * packets counters) are 32-bit registers and could overflow quite in mvpp2_probe()
7583 * quickly. For instance, a 10Gb link used at full bandwidth with the in mvpp2_probe()
7584 * smallest packets (64B) will overflow a 32-bit counter in less than in mvpp2_probe()
7585 * 30 seconds. Then, use a workqueue to fill 64-bit counters. in mvpp2_probe()
7587 snprintf(priv->queue_name, sizeof(priv->queue_name), in mvpp2_probe()
7588 "stats-wq-%s%s", netdev_name(priv->port_list[0]->dev), in mvpp2_probe()
7589 priv->port_count > 1 ? "+" : ""); in mvpp2_probe()
7590 priv->stats_queue = create_singlethread_workqueue(priv->queue_name); in mvpp2_probe()
7591 if (!priv->stats_queue) { in mvpp2_probe()
7592 err = -ENOMEM; in mvpp2_probe()
7596 if (priv->global_tx_fc && priv->hw_version >= MVPP22) { in mvpp2_probe()
7599 …dev_warn(&pdev->dev, "Minimum of CM3 firmware 18.09 and chip revision B0 required for flow control… in mvpp2_probe()
7602 mvpp2_dbgfs_init(priv, pdev->name); in mvpp2_probe()
7612 if (priv->port_list[i]) in mvpp2_probe()
7613 mvpp2_port_remove(priv->port_list[i]); in mvpp2_probe()
7617 clk_disable_unprepare(priv->axi_clk); in mvpp2_probe()
7619 clk_disable_unprepare(priv->mg_core_clk); in mvpp2_probe()
7621 clk_disable_unprepare(priv->mg_clk); in mvpp2_probe()
7623 clk_disable_unprepare(priv->gop_clk); in mvpp2_probe()
7625 clk_disable_unprepare(priv->pp_clk); in mvpp2_probe()
7632 struct fwnode_handle *fwnode = pdev->dev.fwnode; in mvpp2_remove()
7639 if (priv->port_list[i]) { in mvpp2_remove()
7640 mutex_destroy(&priv->port_list[i]->gather_stats_lock); in mvpp2_remove()
7641 mvpp2_port_remove(priv->port_list[i]); in mvpp2_remove()
7646 destroy_workqueue(priv->stats_queue); in mvpp2_remove()
7648 if (priv->percpu_pools) in mvpp2_remove()
7652 struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i]; in mvpp2_remove()
7654 mvpp2_bm_pool_destroy(&pdev->dev, priv, bm_pool); in mvpp2_remove()
7658 struct mvpp2_tx_queue *aggr_txq = &priv->aggr_txqs[i]; in mvpp2_remove()
7660 dma_free_coherent(&pdev->dev, in mvpp2_remove()
7662 aggr_txq->descs, in mvpp2_remove()
7663 aggr_txq->descs_dma); in mvpp2_remove()
7669 clk_disable_unprepare(priv->axi_clk); in mvpp2_remove()
7670 clk_disable_unprepare(priv->mg_core_clk); in mvpp2_remove()
7671 clk_disable_unprepare(priv->mg_clk); in mvpp2_remove()
7672 clk_disable_unprepare(priv->pp_clk); in mvpp2_remove()
7673 clk_disable_unprepare(priv->gop_clk); in mvpp2_remove()
7680 .compatible = "marvell,armada-375-pp2",
7684 .compatible = "marvell,armada-7k-pp22",
7711 MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");