Lines Matching +full:multi +full:- +full:word
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
18 #include "emac-sgmii.h"
210 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
217 /* EMAC Errors in emac_rrd.word[3] */
234 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
235 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
236 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
238 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
239 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
256 /* The HASH Table is an array of 2 32-bit registers. It is in emac_mac_multicast_addr_set()
263 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); in emac_mac_multicast_addr_set()
265 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); in emac_mac_multicast_addr_set()
270 writel(0, adpt->base + EMAC_HASH_TAB_REG0); in emac_mac_multicast_addr_clear()
271 writel(0, adpt->base + EMAC_HASH_TAB_REG1); in emac_mac_multicast_addr_clear()
283 struct net_device *netdev = adpt->netdev; in emac_mac_mode_config()
286 mac = readl(adpt->base + EMAC_MAC_CTRL); in emac_mac_mode_config()
289 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) in emac_mac_mode_config()
292 if (netdev->flags & IFF_PROMISC) in emac_mac_mode_config()
295 if (netdev->flags & IFF_ALLMULTI) in emac_mac_mode_config()
298 writel(mac, adpt->base + EMAC_MAC_CTRL); in emac_mac_mode_config()
305 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr), in emac_mac_dma_rings_config()
306 adpt->base + EMAC_DESC_CTRL_1); in emac_mac_dma_rings_config()
308 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr), in emac_mac_dma_rings_config()
309 adpt->base + EMAC_DESC_CTRL_8); in emac_mac_dma_rings_config()
311 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK, in emac_mac_dma_rings_config()
312 adpt->base + EMAC_DESC_CTRL_9); in emac_mac_dma_rings_config()
315 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr), in emac_mac_dma_rings_config()
316 adpt->base + EMAC_DESC_CTRL_0); in emac_mac_dma_rings_config()
318 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr), in emac_mac_dma_rings_config()
319 adpt->base + EMAC_DESC_CTRL_2); in emac_mac_dma_rings_config()
320 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr), in emac_mac_dma_rings_config()
321 adpt->base + EMAC_DESC_CTRL_5); in emac_mac_dma_rings_config()
323 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK, in emac_mac_dma_rings_config()
324 adpt->base + EMAC_DESC_CTRL_3); in emac_mac_dma_rings_config()
325 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK, in emac_mac_dma_rings_config()
326 adpt->base + EMAC_DESC_CTRL_6); in emac_mac_dma_rings_config()
328 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK, in emac_mac_dma_rings_config()
329 adpt->base + EMAC_DESC_CTRL_4); in emac_mac_dma_rings_config()
331 writel(0, adpt->base + EMAC_DESC_CTRL_11); in emac_mac_dma_rings_config()
336 writel(1, adpt->base + EMAC_INTER_SRAM_PART9); in emac_mac_dma_rings_config()
345 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1); in emac_mac_tx_config()
347 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) & in emac_mac_tx_config()
354 writel(val, adpt->base + EMAC_TXQ_CTRL_0); in emac_mac_tx_config()
355 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2, in emac_mac_tx_config()
364 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) & in emac_mac_rx_config()
368 writel(val, adpt->base + EMAC_RXQ_CTRL_0); in emac_mac_rx_config()
370 val = readl(adpt->base + EMAC_RXQ_CTRL_1); in emac_mac_rx_config()
376 writel(val, adpt->base + EMAC_RXQ_CTRL_1); in emac_mac_rx_config()
378 val = readl(adpt->base + EMAC_RXQ_CTRL_2); in emac_mac_rx_config()
382 writel(val, adpt->base + EMAC_RXQ_CTRL_2); in emac_mac_rx_config()
384 val = readl(adpt->base + EMAC_RXQ_CTRL_3); in emac_mac_rx_config()
387 writel(val, adpt->base + EMAC_RXQ_CTRL_3); in emac_mac_rx_config()
395 switch (adpt->dma_order) { in emac_mac_dma_config()
409 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) & in emac_mac_dma_config()
411 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) & in emac_mac_dma_config()
413 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) & in emac_mac_dma_config()
415 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) & in emac_mac_dma_config()
419 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL); in emac_mac_dma_config()
427 /* for example: 00-A0-C6-11-22-33 in emac_set_mac_address()
428 * 0<-->C6112233, 1<-->00A0. in emac_set_mac_address()
431 /* low 32bit word */ in emac_set_mac_address()
434 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0); in emac_set_mac_address()
436 /* hight 32bit word */ in emac_set_mac_address()
438 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1); in emac_set_mac_address()
443 struct net_device *netdev = adpt->netdev; in emac_mac_config()
447 emac_set_mac_address(adpt, netdev->dev_addr); in emac_mac_config()
449 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; in emac_mac_config()
450 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ? in emac_mac_config()
455 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN, in emac_mac_config()
456 adpt->base + EMAC_MAX_FRAM_LEN_CTRL); in emac_mac_config()
462 val = readl(adpt->base + EMAC_AXI_MAST_CTRL); in emac_mac_config()
465 writel(val, adpt->base + EMAC_AXI_MAST_CTRL); in emac_mac_config()
466 writel(0, adpt->base + EMAC_CLK_GATE_CTRL); in emac_mac_config()
467 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL); in emac_mac_config()
474 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST); in emac_mac_reset()
477 /* interrupt clear-on-read */ in emac_mac_reset()
478 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN); in emac_mac_reset()
483 struct phy_device *phydev = adpt->phydev; in emac_mac_start()
487 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN); in emac_mac_start()
490 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN); in emac_mac_start()
493 mac = readl(adpt->base + EMAC_MAC_CTRL); in emac_mac_start()
494 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1); in emac_mac_start()
504 if (adpt->automatic) { in emac_mac_start()
506 adpt->rx_flow_control = phydev->pause; in emac_mac_start()
507 adpt->tx_flow_control = phydev->pause != phydev->asym_pause; in emac_mac_start()
509 mac |= adpt->rx_flow_control ? RXFC : 0; in emac_mac_start()
510 mac |= adpt->tx_flow_control ? TXFC : 0; in emac_mac_start()
514 if (phydev->speed == SPEED_1000) { in emac_mac_start()
522 if (phydev->duplex == DUPLEX_FULL) in emac_mac_start()
529 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK); in emac_mac_start()
536 /* Enable single-pause-frame mode if requested. in emac_mac_start()
549 * non-stop "flood" of pause frames sometimes has the effect of in emac_mac_start()
553 * The user can enable or disable single-pause-frame mode in emac_mac_start()
556 mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0; in emac_mac_start()
558 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1); in emac_mac_start()
560 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL); in emac_mac_start()
566 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT); in emac_mac_start()
568 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL); in emac_mac_start()
572 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL, in emac_mac_start()
578 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0); in emac_mac_stop()
579 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0); in emac_mac_stop()
580 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0); in emac_mac_stop()
587 struct emac_tx_queue *tx_q = &adpt->tx_q; in emac_tx_q_descs_free()
592 if (!tx_q->tpd.tpbuff) in emac_tx_q_descs_free()
595 for (i = 0; i < tx_q->tpd.count; i++) { in emac_tx_q_descs_free()
598 if (tpbuf->dma_addr) { in emac_tx_q_descs_free()
599 dma_unmap_single(adpt->netdev->dev.parent, in emac_tx_q_descs_free()
600 tpbuf->dma_addr, tpbuf->length, in emac_tx_q_descs_free()
602 tpbuf->dma_addr = 0; in emac_tx_q_descs_free()
604 if (tpbuf->skb) { in emac_tx_q_descs_free()
605 dev_kfree_skb_any(tpbuf->skb); in emac_tx_q_descs_free()
606 tpbuf->skb = NULL; in emac_tx_q_descs_free()
610 size = sizeof(struct emac_buffer) * tx_q->tpd.count; in emac_tx_q_descs_free()
611 memset(tx_q->tpd.tpbuff, 0, size); in emac_tx_q_descs_free()
614 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size); in emac_tx_q_descs_free()
616 tx_q->tpd.consume_idx = 0; in emac_tx_q_descs_free()
617 tx_q->tpd.produce_idx = 0; in emac_tx_q_descs_free()
623 struct device *dev = adpt->netdev->dev.parent; in emac_rx_q_free_descs()
624 struct emac_rx_queue *rx_q = &adpt->rx_q; in emac_rx_q_free_descs()
629 if (!rx_q->rfd.rfbuff) in emac_rx_q_free_descs()
632 for (i = 0; i < rx_q->rfd.count; i++) { in emac_rx_q_free_descs()
635 if (rfbuf->dma_addr) { in emac_rx_q_free_descs()
636 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length, in emac_rx_q_free_descs()
638 rfbuf->dma_addr = 0; in emac_rx_q_free_descs()
640 if (rfbuf->skb) { in emac_rx_q_free_descs()
641 dev_kfree_skb(rfbuf->skb); in emac_rx_q_free_descs()
642 rfbuf->skb = NULL; in emac_rx_q_free_descs()
646 size = sizeof(struct emac_buffer) * rx_q->rfd.count; in emac_rx_q_free_descs()
647 memset(rx_q->rfd.rfbuff, 0, size); in emac_rx_q_free_descs()
650 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size); in emac_rx_q_free_descs()
651 rx_q->rrd.produce_idx = 0; in emac_rx_q_free_descs()
652 rx_q->rrd.consume_idx = 0; in emac_rx_q_free_descs()
654 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size); in emac_rx_q_free_descs()
655 rx_q->rfd.produce_idx = 0; in emac_rx_q_free_descs()
656 rx_q->rfd.consume_idx = 0; in emac_rx_q_free_descs()
662 struct emac_tx_queue *tx_q = &adpt->tx_q; in emac_tx_q_bufs_free()
666 kfree(tx_q->tpd.tpbuff); in emac_tx_q_bufs_free()
667 tx_q->tpd.tpbuff = NULL; in emac_tx_q_bufs_free()
668 tx_q->tpd.v_addr = NULL; in emac_tx_q_bufs_free()
669 tx_q->tpd.dma_addr = 0; in emac_tx_q_bufs_free()
670 tx_q->tpd.size = 0; in emac_tx_q_bufs_free()
677 struct emac_ring_header *ring_header = &adpt->ring_header; in emac_tx_q_desc_alloc()
678 int node = dev_to_node(adpt->netdev->dev.parent); in emac_tx_q_desc_alloc()
681 size = sizeof(struct emac_buffer) * tx_q->tpd.count; in emac_tx_q_desc_alloc()
682 tx_q->tpd.tpbuff = kzalloc_node(size, GFP_KERNEL, node); in emac_tx_q_desc_alloc()
683 if (!tx_q->tpd.tpbuff) in emac_tx_q_desc_alloc()
684 return -ENOMEM; in emac_tx_q_desc_alloc()
686 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4); in emac_tx_q_desc_alloc()
687 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used; in emac_tx_q_desc_alloc()
688 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used; in emac_tx_q_desc_alloc()
689 ring_header->used += ALIGN(tx_q->tpd.size, 8); in emac_tx_q_desc_alloc()
690 tx_q->tpd.produce_idx = 0; in emac_tx_q_desc_alloc()
691 tx_q->tpd.consume_idx = 0; in emac_tx_q_desc_alloc()
699 struct emac_rx_queue *rx_q = &adpt->rx_q; in emac_rx_q_bufs_free()
703 kfree(rx_q->rfd.rfbuff); in emac_rx_q_bufs_free()
704 rx_q->rfd.rfbuff = NULL; in emac_rx_q_bufs_free()
706 rx_q->rfd.v_addr = NULL; in emac_rx_q_bufs_free()
707 rx_q->rfd.dma_addr = 0; in emac_rx_q_bufs_free()
708 rx_q->rfd.size = 0; in emac_rx_q_bufs_free()
710 rx_q->rrd.v_addr = NULL; in emac_rx_q_bufs_free()
711 rx_q->rrd.dma_addr = 0; in emac_rx_q_bufs_free()
712 rx_q->rrd.size = 0; in emac_rx_q_bufs_free()
718 struct emac_ring_header *ring_header = &adpt->ring_header; in emac_rx_descs_alloc()
719 int node = dev_to_node(adpt->netdev->dev.parent); in emac_rx_descs_alloc()
720 struct emac_rx_queue *rx_q = &adpt->rx_q; in emac_rx_descs_alloc()
723 size = sizeof(struct emac_buffer) * rx_q->rfd.count; in emac_rx_descs_alloc()
724 rx_q->rfd.rfbuff = kzalloc_node(size, GFP_KERNEL, node); in emac_rx_descs_alloc()
725 if (!rx_q->rfd.rfbuff) in emac_rx_descs_alloc()
726 return -ENOMEM; in emac_rx_descs_alloc()
728 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4); in emac_rx_descs_alloc()
729 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4); in emac_rx_descs_alloc()
731 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used; in emac_rx_descs_alloc()
732 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used; in emac_rx_descs_alloc()
733 ring_header->used += ALIGN(rx_q->rrd.size, 8); in emac_rx_descs_alloc()
735 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used; in emac_rx_descs_alloc()
736 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used; in emac_rx_descs_alloc()
737 ring_header->used += ALIGN(rx_q->rfd.size, 8); in emac_rx_descs_alloc()
739 rx_q->rrd.produce_idx = 0; in emac_rx_descs_alloc()
740 rx_q->rrd.consume_idx = 0; in emac_rx_descs_alloc()
742 rx_q->rfd.produce_idx = 0; in emac_rx_descs_alloc()
743 rx_q->rfd.consume_idx = 0; in emac_rx_descs_alloc()
751 struct emac_ring_header *ring_header = &adpt->ring_header; in emac_mac_rx_tx_rings_alloc_all()
752 struct device *dev = adpt->netdev->dev.parent; in emac_mac_rx_tx_rings_alloc_all()
753 unsigned int num_tx_descs = adpt->tx_desc_cnt; in emac_mac_rx_tx_rings_alloc_all()
754 unsigned int num_rx_descs = adpt->rx_desc_cnt; in emac_mac_rx_tx_rings_alloc_all()
757 adpt->tx_q.tpd.count = adpt->tx_desc_cnt; in emac_mac_rx_tx_rings_alloc_all()
759 adpt->rx_q.rrd.count = adpt->rx_desc_cnt; in emac_mac_rx_tx_rings_alloc_all()
760 adpt->rx_q.rfd.count = adpt->rx_desc_cnt; in emac_mac_rx_tx_rings_alloc_all()
765 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) + in emac_mac_rx_tx_rings_alloc_all()
766 num_rx_descs * (adpt->rfd_size * 4) + in emac_mac_rx_tx_rings_alloc_all()
767 num_rx_descs * (adpt->rrd_size * 4) + in emac_mac_rx_tx_rings_alloc_all()
770 ring_header->used = 0; in emac_mac_rx_tx_rings_alloc_all()
771 ring_header->v_addr = dma_alloc_coherent(dev, ring_header->size, in emac_mac_rx_tx_rings_alloc_all()
772 &ring_header->dma_addr, in emac_mac_rx_tx_rings_alloc_all()
774 if (!ring_header->v_addr) in emac_mac_rx_tx_rings_alloc_all()
775 return -ENOMEM; in emac_mac_rx_tx_rings_alloc_all()
777 ring_header->used = ALIGN(ring_header->dma_addr, 8) - in emac_mac_rx_tx_rings_alloc_all()
778 ring_header->dma_addr; in emac_mac_rx_tx_rings_alloc_all()
780 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q); in emac_mac_rx_tx_rings_alloc_all()
782 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n"); in emac_mac_rx_tx_rings_alloc_all()
788 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n"); in emac_mac_rx_tx_rings_alloc_all()
797 dma_free_coherent(dev, ring_header->size, in emac_mac_rx_tx_rings_alloc_all()
798 ring_header->v_addr, ring_header->dma_addr); in emac_mac_rx_tx_rings_alloc_all()
800 ring_header->v_addr = NULL; in emac_mac_rx_tx_rings_alloc_all()
801 ring_header->dma_addr = 0; in emac_mac_rx_tx_rings_alloc_all()
802 ring_header->size = 0; in emac_mac_rx_tx_rings_alloc_all()
803 ring_header->used = 0; in emac_mac_rx_tx_rings_alloc_all()
811 struct emac_ring_header *ring_header = &adpt->ring_header; in emac_mac_rx_tx_rings_free_all()
812 struct device *dev = adpt->netdev->dev.parent; in emac_mac_rx_tx_rings_free_all()
817 dma_free_coherent(dev, ring_header->size, in emac_mac_rx_tx_rings_free_all()
818 ring_header->v_addr, ring_header->dma_addr); in emac_mac_rx_tx_rings_free_all()
820 ring_header->v_addr = NULL; in emac_mac_rx_tx_rings_free_all()
821 ring_header->dma_addr = 0; in emac_mac_rx_tx_rings_free_all()
822 ring_header->size = 0; in emac_mac_rx_tx_rings_free_all()
823 ring_header->used = 0; in emac_mac_rx_tx_rings_free_all()
831 adpt->tx_q.tpd.produce_idx = 0; in emac_mac_rx_tx_ring_reset_all()
832 adpt->tx_q.tpd.consume_idx = 0; in emac_mac_rx_tx_ring_reset_all()
833 for (i = 0; i < adpt->tx_q.tpd.count; i++) in emac_mac_rx_tx_ring_reset_all()
834 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0; in emac_mac_rx_tx_ring_reset_all()
836 adpt->rx_q.rrd.produce_idx = 0; in emac_mac_rx_tx_ring_reset_all()
837 adpt->rx_q.rrd.consume_idx = 0; in emac_mac_rx_tx_ring_reset_all()
838 adpt->rx_q.rfd.produce_idx = 0; in emac_mac_rx_tx_ring_reset_all()
839 adpt->rx_q.rfd.consume_idx = 0; in emac_mac_rx_tx_ring_reset_all()
840 for (i = 0; i < adpt->rx_q.rfd.count; i++) in emac_mac_rx_tx_ring_reset_all()
841 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0; in emac_mac_rx_tx_ring_reset_all()
849 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx); in emac_mac_rx_rfd_create()
854 if (++rx_q->rfd.produce_idx == rx_q->rfd.count) in emac_mac_rx_rfd_create()
855 rx_q->rfd.produce_idx = 0; in emac_mac_rx_rfd_create()
867 next_produce_idx = rx_q->rfd.produce_idx + 1; in emac_mac_rx_descs_refill()
868 if (next_produce_idx == rx_q->rfd.count) in emac_mac_rx_descs_refill()
871 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); in emac_mac_rx_descs_refill()
875 while (!next_rxbuf->dma_addr) { in emac_mac_rx_descs_refill()
879 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size); in emac_mac_rx_descs_refill()
883 curr_rxbuf->dma_addr = in emac_mac_rx_descs_refill()
884 dma_map_single(adpt->netdev->dev.parent, skb->data, in emac_mac_rx_descs_refill()
885 adpt->rxbuf_size, DMA_FROM_DEVICE); in emac_mac_rx_descs_refill()
887 ret = dma_mapping_error(adpt->netdev->dev.parent, in emac_mac_rx_descs_refill()
888 curr_rxbuf->dma_addr); in emac_mac_rx_descs_refill()
893 curr_rxbuf->skb = skb; in emac_mac_rx_descs_refill()
894 curr_rxbuf->length = adpt->rxbuf_size; in emac_mac_rx_descs_refill()
896 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr); in emac_mac_rx_descs_refill()
897 next_produce_idx = rx_q->rfd.produce_idx + 1; in emac_mac_rx_descs_refill()
898 if (next_produce_idx == rx_q->rfd.count) in emac_mac_rx_descs_refill()
901 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); in emac_mac_rx_descs_refill()
907 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) & in emac_mac_rx_descs_refill()
908 rx_q->produce_mask; in emac_mac_rx_descs_refill()
909 emac_reg_update32(adpt->base + rx_q->produce_reg, in emac_mac_rx_descs_refill()
910 rx_q->produce_mask, prod_idx); in emac_mac_rx_descs_refill()
917 struct phy_device *phydev = netdev->phydev; in emac_adjust_link()
919 if (phydev->link) { in emac_adjust_link()
933 struct net_device *netdev = adpt->netdev; in emac_mac_up()
938 emac_mac_rx_descs_refill(adpt, &adpt->rx_q); in emac_mac_up()
940 adpt->phydev->irq = PHY_POLL; in emac_mac_up()
941 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link, in emac_mac_up()
944 netdev_err(adpt->netdev, "could not connect phy\n"); in emac_mac_up()
948 phy_attached_print(adpt->phydev, NULL); in emac_mac_up()
951 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS); in emac_mac_up()
952 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK); in emac_mac_up()
954 phy_start(adpt->phydev); in emac_mac_up()
956 napi_enable(&adpt->rx_q.napi); in emac_mac_up()
965 struct net_device *netdev = adpt->netdev; in emac_mac_down()
968 napi_disable(&adpt->rx_q.napi); in emac_mac_down()
970 phy_stop(adpt->phydev); in emac_mac_down()
976 writel(DIS_INT, adpt->base + EMAC_INT_STATUS); in emac_mac_down()
977 writel(0, adpt->base + EMAC_INT_MASK); in emac_mac_down()
978 synchronize_irq(adpt->irq.irq); in emac_mac_down()
980 phy_disconnect(adpt->phydev); in emac_mac_down()
985 netdev_reset_queue(adpt->netdev); in emac_mac_down()
994 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx); in emac_rx_process_rrd()
996 rrd->word[3] = *(hw_rrd + 3); in emac_rx_process_rrd()
1001 rrd->word[4] = 0; in emac_rx_process_rrd()
1002 rrd->word[5] = 0; in emac_rx_process_rrd()
1004 rrd->word[0] = *(hw_rrd++); in emac_rx_process_rrd()
1005 rrd->word[1] = *(hw_rrd++); in emac_rx_process_rrd()
1006 rrd->word[2] = *(hw_rrd++); in emac_rx_process_rrd()
1009 netdev_err(adpt->netdev, in emac_rx_process_rrd()
1010 "error: multi-RFD not support yet! nor:%lu\n", in emac_rx_process_rrd()
1016 *hw_rrd = rrd->word[3]; in emac_rx_process_rrd()
1018 if (++rx_q->rrd.consume_idx == rx_q->rrd.count) in emac_rx_process_rrd()
1019 rx_q->rrd.consume_idx = 0; in emac_rx_process_rrd()
1030 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx; in emac_tx_tpd_create()
1031 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx); in emac_tx_tpd_create()
1033 if (++tx_q->tpd.produce_idx == tx_q->tpd.count) in emac_tx_tpd_create()
1034 tx_q->tpd.produce_idx = 0; in emac_tx_tpd_create()
1036 *(hw_tpd++) = tpd->word[0]; in emac_tx_tpd_create()
1037 *(hw_tpd++) = tpd->word[1]; in emac_tx_tpd_create()
1038 *(hw_tpd++) = tpd->word[2]; in emac_tx_tpd_create()
1039 *hw_tpd = tpd->word[3]; in emac_tx_tpd_create()
1047 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx); in emac_tx_tpd_mark_last()
1057 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff; in emac_rx_rfd_clean()
1063 if (++consume_idx == rx_q->rfd.count) in emac_rx_rfd_clean()
1067 rx_q->rfd.consume_idx = consume_idx; in emac_rx_rfd_clean()
1068 rx_q->rfd.process_idx = consume_idx; in emac_rx_rfd_clean()
1083 napi_gro_receive(&rx_q->napi, skb); in emac_receive_skb()
1091 struct net_device *netdev = adpt->netdev; in emac_mac_rx_process()
1098 reg = readl_relaxed(adpt->base + rx_q->consume_reg); in emac_mac_rx_process()
1100 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift; in emac_mac_rx_process()
1101 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ? in emac_mac_rx_process()
1102 (hw_consume_idx - rx_q->rrd.consume_idx) : in emac_mac_rx_process()
1103 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx); in emac_mac_rx_process()
1115 dma_unmap_single(adpt->netdev->dev.parent, in emac_mac_rx_process()
1116 rfbuf->dma_addr, rfbuf->length, in emac_mac_rx_process()
1118 rfbuf->dma_addr = 0; in emac_mac_rx_process()
1119 skb = rfbuf->skb; in emac_mac_rx_process()
1121 netdev_err(adpt->netdev, in emac_mac_rx_process()
1122 "error: multi-RFD not support yet!\n"); in emac_mac_rx_process()
1126 num_consume_pkts--; in emac_mac_rx_process()
1133 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) { in emac_mac_rx_process()
1134 netif_dbg(adpt, rx_status, adpt->netdev, in emac_mac_rx_process()
1136 rrd.word[0], rrd.word[1], in emac_mac_rx_process()
1137 rrd.word[2], rrd.word[3]); in emac_mac_rx_process()
1143 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN); in emac_mac_rx_process()
1144 skb->dev = netdev; in emac_mac_rx_process()
1145 skb->protocol = eth_type_trans(skb, skb->dev); in emac_mac_rx_process()
1146 if (netdev->features & NETIF_F_RXCSUM) in emac_mac_rx_process()
1147 skb->ip_summed = RRD_L4F(&rrd) ? in emac_mac_rx_process()
1159 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) & in emac_mac_rx_process()
1160 rx_q->process_mask; in emac_mac_rx_process()
1161 emac_reg_update32(adpt->base + rx_q->process_reg, in emac_mac_rx_process()
1162 rx_q->process_mask, proc_idx); in emac_mac_rx_process()
1170 u32 produce_idx = tx_q->tpd.produce_idx; in emac_tpd_num_free_descs()
1171 u32 consume_idx = tx_q->tpd.consume_idx; in emac_tpd_num_free_descs()
1174 (consume_idx - produce_idx - 1) : in emac_tpd_num_free_descs()
1175 (tx_q->tpd.count + consume_idx - produce_idx - 1); in emac_tpd_num_free_descs()
1181 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg); in emac_mac_tx_process()
1185 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift; in emac_mac_tx_process()
1187 while (tx_q->tpd.consume_idx != hw_consume_idx) { in emac_mac_tx_process()
1188 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx); in emac_mac_tx_process()
1189 if (tpbuf->dma_addr) { in emac_mac_tx_process()
1190 dma_unmap_page(adpt->netdev->dev.parent, in emac_mac_tx_process()
1191 tpbuf->dma_addr, tpbuf->length, in emac_mac_tx_process()
1193 tpbuf->dma_addr = 0; in emac_mac_tx_process()
1196 if (tpbuf->skb) { in emac_mac_tx_process()
1198 bytes_compl += tpbuf->skb->len; in emac_mac_tx_process()
1199 dev_consume_skb_irq(tpbuf->skb); in emac_mac_tx_process()
1200 tpbuf->skb = NULL; in emac_mac_tx_process()
1203 if (++tx_q->tpd.consume_idx == tx_q->tpd.count) in emac_mac_tx_process()
1204 tx_q->tpd.consume_idx = 0; in emac_mac_tx_process()
1207 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl); in emac_mac_tx_process()
1209 if (netif_queue_stopped(adpt->netdev)) in emac_mac_tx_process()
1211 netif_wake_queue(adpt->netdev); in emac_mac_tx_process()
1218 adpt->rx_q.netdev = adpt->netdev; in emac_mac_rx_tx_ring_init_all()
1220 adpt->rx_q.produce_reg = EMAC_MAILBOX_0; in emac_mac_rx_tx_ring_init_all()
1221 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK; in emac_mac_rx_tx_ring_init_all()
1222 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT; in emac_mac_rx_tx_ring_init_all()
1224 adpt->rx_q.process_reg = EMAC_MAILBOX_0; in emac_mac_rx_tx_ring_init_all()
1225 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK; in emac_mac_rx_tx_ring_init_all()
1226 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT; in emac_mac_rx_tx_ring_init_all()
1228 adpt->rx_q.consume_reg = EMAC_MAILBOX_3; in emac_mac_rx_tx_ring_init_all()
1229 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK; in emac_mac_rx_tx_ring_init_all()
1230 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT; in emac_mac_rx_tx_ring_init_all()
1232 adpt->rx_q.irq = &adpt->irq; in emac_mac_rx_tx_ring_init_all()
1233 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT; in emac_mac_rx_tx_ring_init_all()
1235 adpt->tx_q.produce_reg = EMAC_MAILBOX_15; in emac_mac_rx_tx_ring_init_all()
1236 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK; in emac_mac_rx_tx_ring_init_all()
1237 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT; in emac_mac_rx_tx_ring_init_all()
1239 adpt->tx_q.consume_reg = EMAC_MAILBOX_2; in emac_mac_rx_tx_ring_init_all()
1240 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK; in emac_mac_rx_tx_ring_init_all()
1241 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT; in emac_mac_rx_tx_ring_init_all()
1260 if (skb->protocol == htons(ETH_P_IP)) { in emac_tso_csum()
1261 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data) in emac_tso_csum()
1262 + ntohs(ip_hdr(skb)->tot_len); in emac_tso_csum()
1263 if (skb->len > pkt_len) in emac_tso_csum()
1268 if (unlikely(skb->len == hdr_len)) { in emac_tso_csum()
1270 netif_warn(adpt, tx_err, adpt->netdev, in emac_tso_csum()
1275 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { in emac_tso_csum()
1276 ip_hdr(skb)->check = 0; in emac_tso_csum()
1277 tcp_hdr(skb)->check = in emac_tso_csum()
1278 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, in emac_tso_csum()
1279 ip_hdr(skb)->daddr, in emac_tso_csum()
1284 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) { in emac_tso_csum()
1293 TPD_PKT_LEN_SET(&extra_tpd, skb->len); in emac_tso_csum()
1302 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size); in emac_tso_csum()
1307 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { in emac_tso_csum()
1312 netdev_err(adpt->netdev, in emac_tso_csum()
1314 return -EINVAL; in emac_tso_csum()
1316 css = cso + skb->csum_offset; in emac_tso_csum()
1331 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; in emac_tx_fill_tpd()
1332 unsigned int first = tx_q->tpd.produce_idx; in emac_tx_fill_tpd()
1344 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); in emac_tx_fill_tpd()
1345 tpbuf->length = mapped_len; in emac_tx_fill_tpd()
1346 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1347 virt_to_page(skb->data), in emac_tx_fill_tpd()
1348 offset_in_page(skb->data), in emac_tx_fill_tpd()
1349 tpbuf->length, in emac_tx_fill_tpd()
1351 ret = dma_mapping_error(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1352 tpbuf->dma_addr); in emac_tx_fill_tpd()
1356 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1357 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1358 TPD_BUF_LEN_SET(tpd, tpbuf->length); in emac_tx_fill_tpd()
1364 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); in emac_tx_fill_tpd()
1365 tpbuf->length = len - mapped_len; in emac_tx_fill_tpd()
1366 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1367 virt_to_page(skb->data + in emac_tx_fill_tpd()
1369 offset_in_page(skb->data + in emac_tx_fill_tpd()
1371 tpbuf->length, DMA_TO_DEVICE); in emac_tx_fill_tpd()
1372 ret = dma_mapping_error(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1373 tpbuf->dma_addr); in emac_tx_fill_tpd()
1377 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1378 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1379 TPD_BUF_LEN_SET(tpd, tpbuf->length); in emac_tx_fill_tpd()
1385 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in emac_tx_fill_tpd()
1387 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); in emac_tx_fill_tpd()
1388 tpbuf->length = skb_frag_size(frag); in emac_tx_fill_tpd()
1389 tpbuf->dma_addr = skb_frag_dma_map(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1390 frag, 0, tpbuf->length, in emac_tx_fill_tpd()
1392 ret = dma_mapping_error(adpt->netdev->dev.parent, in emac_tx_fill_tpd()
1393 tpbuf->dma_addr); in emac_tx_fill_tpd()
1397 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1398 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); in emac_tx_fill_tpd()
1399 TPD_BUF_LEN_SET(tpd, tpbuf->length); in emac_tx_fill_tpd()
1411 tpbuf->skb = skb; in emac_tx_fill_tpd()
1417 tx_q->tpd.produce_idx = first; in emac_tx_fill_tpd()
1419 while (count--) { in emac_tx_fill_tpd()
1421 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr, in emac_tx_fill_tpd()
1422 tpbuf->length, DMA_TO_DEVICE); in emac_tx_fill_tpd()
1423 tpbuf->dma_addr = 0; in emac_tx_fill_tpd()
1424 tpbuf->length = 0; in emac_tx_fill_tpd()
1426 if (++first == tx_q->tpd.count) in emac_tx_fill_tpd()
1461 netdev_sent_queue(adpt->netdev, skb->len); in emac_mac_tx_buf_send()
1464 * maximum-sized SKB. We need one desc for each fragment, in emac_mac_tx_buf_send()
1469 netif_stop_queue(adpt->netdev); in emac_mac_tx_buf_send()
1472 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) & in emac_mac_tx_buf_send()
1473 tx_q->produce_mask; in emac_mac_tx_buf_send()
1474 emac_reg_update32(adpt->base + tx_q->produce_reg, in emac_mac_tx_buf_send()
1475 tx_q->produce_mask, prod_idx); in emac_mac_tx_buf_send()