24 #include "cpsw.h"
33 int (*cpsw_slave_index)(struct cpsw_common *cpsw, struct cpsw_priv *priv);
35 void cpsw_intr_enable(struct cpsw_common *cpsw)  in cpsw_intr_enable()  argument
37 	writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);  in cpsw_intr_enable()
38 	writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);  in cpsw_intr_enable()
40 	cpdma_ctlr_int_ctrl(cpsw->dma, true);  in cpsw_intr_enable()
43 void cpsw_intr_disable(struct cpsw_common *cpsw)  in cpsw_intr_disable()  argument
45 	writel_relaxed(0, &cpsw->wr_regs->tx_en);  in cpsw_intr_disable()
46 	writel_relaxed(0, &cpsw->wr_regs->rx_en);  in cpsw_intr_disable()
48 	cpdma_ctlr_int_ctrl(cpsw->dma, false);  in cpsw_intr_disable()
87 	struct cpsw_common *cpsw = dev_id;  in cpsw_tx_interrupt()  local
89 	writel(0, &cpsw->wr_regs->tx_en);  in cpsw_tx_interrupt()
90 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);  in cpsw_tx_interrupt()
92 	if (cpsw->quirk_irq) {  in cpsw_tx_interrupt()
93 		disable_irq_nosync(cpsw->irqs_table[1]);  in cpsw_tx_interrupt()
94 		cpsw->tx_irq_disabled = true;  in cpsw_tx_interrupt()
97 	napi_schedule(&cpsw->napi_tx);  in cpsw_tx_interrupt()
103 	struct cpsw_common *cpsw = dev_id;  in cpsw_rx_interrupt()  local
105 	writel(0, &cpsw->wr_regs->rx_en);  in cpsw_rx_interrupt()
106 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);  in cpsw_rx_interrupt()
108 	if (cpsw->quirk_irq) {  in cpsw_rx_interrupt()
109 		disable_irq_nosync(cpsw->irqs_table[0]);  in cpsw_rx_interrupt()
110 		cpsw->rx_irq_disabled = true;  in cpsw_rx_interrupt()
113 	napi_schedule(&cpsw->napi_rx);  in cpsw_rx_interrupt()
119 	struct cpsw_common *cpsw = dev_id;  in cpsw_misc_interrupt()  local
121 	writel(0, &cpsw->wr_regs->misc_en);  in cpsw_misc_interrupt()
122 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_MISC);  in cpsw_misc_interrupt()
123 	cpts_misc_interrupt(cpsw->cpts);  in cpsw_misc_interrupt()
124 	writel(0x10, &cpsw->wr_regs->misc_en);  in cpsw_misc_interrupt()
131 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_tx);  in cpsw_tx_mq_poll()  local
137 	ch_map = cpdma_ctrl_txchs_state(cpsw->dma);  in cpsw_tx_mq_poll()
142 		txv = &cpsw->txv[ch];  in cpsw_tx_mq_poll()
155 		writel(0xff, &cpsw->wr_regs->tx_en);  in cpsw_tx_mq_poll()
163 	struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);  in cpsw_tx_poll()  local
166 	num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);  in cpsw_tx_poll()
169 		writel(0xff, &cpsw->wr_regs->tx_en);  in cpsw_tx_poll()
170 		if (cpsw->tx_irq_disabled) {  in cpsw_tx_poll()
171 			cpsw->tx_irq_disabled = false;  in cpsw_tx_poll()
172 			enable_irq(cpsw->irqs_table[1]);  in cpsw_tx_poll()
181 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_rx);  in cpsw_rx_mq_poll()  local
187 	ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);  in cpsw_rx_mq_poll()
192 		rxv = &cpsw->rxv[ch];  in cpsw_rx_mq_poll()
205 		writel(0xff, &cpsw->wr_regs->rx_en);  in cpsw_rx_mq_poll()
213 	struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);  in cpsw_rx_poll()  local
216 	num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);  in cpsw_rx_poll()
219 		writel(0xff, &cpsw->wr_regs->rx_en);  in cpsw_rx_poll()
220 		if (cpsw->rx_irq_disabled) {  in cpsw_rx_poll()
221 			cpsw->rx_irq_disabled = false;  in cpsw_rx_poll()
222 			enable_irq(cpsw->irqs_table[0]);  in cpsw_rx_poll()
233 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_rx_vlan_encap()  local
255 	if (!cpsw_ale_get_vlan_p0_untag(cpsw->ale, vid)) {  in cpsw_rx_vlan_encap()
292 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_tx_timeout()  local
297 	cpsw_intr_disable(cpsw);  in cpsw_ndo_tx_timeout()
298 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {  in cpsw_ndo_tx_timeout()
299 		cpdma_chan_stop(cpsw->txv[ch].ch);  in cpsw_ndo_tx_timeout()
300 		cpdma_chan_start(cpsw->txv[ch].ch);  in cpsw_ndo_tx_timeout()
303 	cpsw_intr_enable(cpsw);  in cpsw_ndo_tx_timeout()
308 static int cpsw_get_common_speed(struct cpsw_common *cpsw)  in cpsw_get_common_speed()  argument
312 	for (i = 0, speed = 0; i < cpsw->data.slaves; i++)  in cpsw_get_common_speed()
313 		if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)  in cpsw_get_common_speed()
314 			speed += cpsw->slaves[i].phy->speed;  in cpsw_get_common_speed()
319 int cpsw_need_resplit(struct cpsw_common *cpsw)  in cpsw_need_resplit()  argument
325 	speed = cpsw_get_common_speed(cpsw);  in cpsw_need_resplit()
326 	if (speed == cpsw->speed || !speed)  in cpsw_need_resplit()
329 	cpsw->speed = speed;  in cpsw_need_resplit()
331 	for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_need_resplit()
332 		ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);  in cpsw_need_resplit()
340 	if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)  in cpsw_need_resplit()
346 void cpsw_split_res(struct cpsw_common *cpsw)  in cpsw_split_res()  argument
349 	struct cpsw_vector *txv = cpsw->txv;  in cpsw_split_res()
355 	for (i = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_split_res()
364 	if (cpsw->tx_ch_num == rlim_ch_num) {  in cpsw_split_res()
367 		ch_budget = NAPI_POLL_WEIGHT / cpsw->tx_ch_num;  in cpsw_split_res()
371 		max_rate = cpsw->speed * 1000;  in cpsw_split_res()
384 			    (cpsw->tx_ch_num - rlim_ch_num);  in cpsw_split_res()
386 			      (cpsw->tx_ch_num - rlim_ch_num);  in cpsw_split_res()
391 	for (i = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_split_res()
405 			cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);  in cpsw_split_res()
410 			cpdma_chan_set_weight(cpsw->txv[i].ch, 0);  in cpsw_split_res()
421 	ch_budget = budget / cpsw->rx_ch_num;  in cpsw_split_res()
422 	for (i = 0; i < cpsw->rx_ch_num; i++) {  in cpsw_split_res()
423 		cpsw->rxv[i].budget = ch_budget;  in cpsw_split_res()
428 		cpsw->rxv[0].budget += budget;  in cpsw_split_res()
431 int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs,  in cpsw_init_common()  argument
439 	struct device *dev = cpsw->dev;  in cpsw_init_common()
444 	data = &cpsw->data;  in cpsw_init_common()
445 	cpsw->rx_ch_num = 1;  in cpsw_init_common()
446 	cpsw->tx_ch_num = 1;  in cpsw_init_common()
448 	cpsw->version = readl(&cpsw->regs->id_ver);  in cpsw_init_common()
453 	switch (cpsw->version) {  in cpsw_init_common()
455 		cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;  in cpsw_init_common()
457 		cpsw->hw_stats	     = ss_regs + CPSW1_HW_STATS;  in cpsw_init_common()
469 		cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;  in cpsw_init_common()
471 		cpsw->hw_stats	     = ss_regs + CPSW2_HW_STATS;  in cpsw_init_common()
481 		dev_err(dev, "unknown version 0x%08x\n", cpsw->version);  in cpsw_init_common()
485 	for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_init_common()
486 		struct cpsw_slave *slave = &cpsw->slaves[i];  in cpsw_init_common()
487 		void __iomem		*regs = cpsw->regs;  in cpsw_init_common()
490 		slave->data	= &cpsw->data.slave_data[i];  in cpsw_init_common()
493 		slave->mac_sl = cpsw_sl_get("cpsw", dev, regs + sliver_offset);  in cpsw_init_common()
504 	ale_params.dev_id		= "cpsw";  in cpsw_init_common()
505 	ale_params.bus_freq		= cpsw->bus_freq_mhz * 1000000;  in cpsw_init_common()
507 	cpsw->ale = cpsw_ale_create(&ale_params);  in cpsw_init_common()
508 	if (IS_ERR(cpsw->ale)) {  in cpsw_init_common()
510 		return PTR_ERR(cpsw->ale);  in cpsw_init_common()
527 	dma_params.bus_freq_mhz		= cpsw->bus_freq_mhz;  in cpsw_init_common()
530 	cpsw->dma = cpdma_ctlr_create(&dma_params);  in cpsw_init_common()
531 	if (!cpsw->dma) {  in cpsw_init_common()
536 	cpts_node = of_get_child_by_name(cpsw->dev->of_node, "cpts");  in cpsw_init_common()
538 		cpts_node = cpsw->dev->of_node;  in cpsw_init_common()
540 	cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpts_node,  in cpsw_init_common()
542 	if (IS_ERR(cpsw->cpts)) {  in cpsw_init_common()
543 		ret = PTR_ERR(cpsw->cpts);  in cpsw_init_common()
544 		cpdma_ctlr_destroy(cpsw->dma);  in cpsw_init_common()
555 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_v1()  local
556 	struct cpsw_slave *slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_hwtstamp_v1()
579 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_v2()  local
583 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_hwtstamp_v2()
586 	switch (cpsw->version) {  in cpsw_hwtstamp_v2()
612 	writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);  in cpsw_hwtstamp_v2()
613 	writel_relaxed(ETH_P_8021Q, &cpsw->regs->vlan_ltype);  in cpsw_hwtstamp_v2()
619 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_set()  local
622 	if (cpsw->version != CPSW_VERSION_1 &&  in cpsw_hwtstamp_set()
623 	    cpsw->version != CPSW_VERSION_2 &&  in cpsw_hwtstamp_set()
624 	    cpsw->version != CPSW_VERSION_3)  in cpsw_hwtstamp_set()
661 	switch (cpsw->version) {  in cpsw_hwtstamp_set()
678 	struct cpsw_common *cpsw = ndev_to_cpsw(dev);  in cpsw_hwtstamp_get()  local
682 	if (cpsw->version != CPSW_VERSION_1 &&  in cpsw_hwtstamp_get()
683 	    cpsw->version != CPSW_VERSION_2 &&  in cpsw_hwtstamp_get()
684 	    cpsw->version != CPSW_VERSION_3)  in cpsw_hwtstamp_get()
708 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_ioctl()  local
709 	int slave_no = cpsw_slave_index(cpsw, priv);  in cpsw_ndo_ioctl()
715 	phy = cpsw->slaves[slave_no].phy;  in cpsw_ndo_ioctl()
735 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_set_tx_maxrate()  local
746 	min_rate = cpdma_chan_get_min_rate(cpsw->dma);  in cpsw_ndo_set_tx_maxrate()
753 	if (rate > cpsw->speed) {  in cpsw_ndo_set_tx_maxrate()
758 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_ndo_set_tx_maxrate()
762 	ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);  in cpsw_ndo_set_tx_maxrate()
763 	pm_runtime_put(cpsw->dev);  in cpsw_ndo_set_tx_maxrate()
769 	for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_ndo_set_tx_maxrate()
770 		slave = &cpsw->slaves[i];  in cpsw_ndo_set_tx_maxrate()
777 	cpsw_split_res(cpsw);  in cpsw_ndo_set_tx_maxrate()
791 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_shp_is_off()  local
795 	val = readl_relaxed(&cpsw->regs->ptype);  in cpsw_shp_is_off()
797 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_shp_is_off()
807 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_fifo_shp_on()  local
811 	val = readl_relaxed(&cpsw->regs->ptype);  in cpsw_fifo_shp_on()
813 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_fifo_shp_on()
818 	writel_relaxed(val, &cpsw->regs->ptype);  in cpsw_fifo_shp_on()
823 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_fifo_bw()  local
834 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_fifo_bw()
883 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_fifo_rlimit()  local
892 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_fifo_rlimit()
893 	tx_in_ctl_rg = cpsw->version == CPSW_VERSION_1 ?  in cpsw_set_fifo_rlimit()
935 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_cbs()  local
958 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_cbs()
972 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_set_cbs()
987 	pm_runtime_put_sync(cpsw->dev);  in cpsw_set_cbs()
995 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_mqprio()  local
1008 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_set_mqprio()
1035 	offset = cpsw->version == CPSW_VERSION_1 ?  in cpsw_set_mqprio()
1038 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_mqprio()
1041 	pm_runtime_put_sync(cpsw->dev);  in cpsw_set_mqprio()
1081 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_mqprio_resume()  local
1095 	tx_prio_rg = cpsw->version == CPSW_VERSION_1 ?  in cpsw_mqprio_resume()
1103 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_fill_rx_channels()  local
1111 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_fill_rx_channels()
1112 		pool = cpsw->page_pool[ch];  in cpsw_fill_rx_channels()
1113 		ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);  in cpsw_fill_rx_channels()
1126 			ret = cpdma_chan_idle_submit_mapped(cpsw->rxv[ch].ch,  in cpsw_fill_rx_channels()
1128 							    cpsw->rx_packet_max,  in cpsw_fill_rx_channels()
1146 static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw,  in cpsw_create_page_pool()  argument
1157 	pp_params.dev = cpsw->dev;  in cpsw_create_page_pool()
1161 		dev_err(cpsw->dev, "cannot create rx page pool\n");  in cpsw_create_page_pool()
1166 static int cpsw_create_rx_pool(struct cpsw_common *cpsw, int ch)  in cpsw_create_rx_pool()  argument
1171 	pool_size = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);  in cpsw_create_rx_pool()
1172 	pool = cpsw_create_page_pool(cpsw, pool_size);  in cpsw_create_rx_pool()
1176 		cpsw->page_pool[ch] = pool;  in cpsw_create_rx_pool()
1183 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndev_create_xdp_rxq()  local
1188 	pool = cpsw->page_pool[ch];  in cpsw_ndev_create_xdp_rxq()
1212 void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw)  in cpsw_destroy_xdp_rxqs()  argument
1217 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_destroy_xdp_rxqs()
1218 		for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_destroy_xdp_rxqs()
1219 			ndev = cpsw->slaves[i].ndev;  in cpsw_destroy_xdp_rxqs()
1226 		page_pool_destroy(cpsw->page_pool[ch]);  in cpsw_destroy_xdp_rxqs()
1227 		cpsw->page_pool[ch] = NULL;  in cpsw_destroy_xdp_rxqs()
1231 int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw)  in cpsw_create_xdp_rxqs()  argument
1236 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_create_xdp_rxqs()
1237 		ret = cpsw_create_rx_pool(cpsw, ch);  in cpsw_create_xdp_rxqs()
1244 		for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_create_xdp_rxqs()
1245 			ndev = cpsw->slaves[i].ndev;  in cpsw_create_xdp_rxqs()
1258 	cpsw_destroy_xdp_rxqs(cpsw);  in cpsw_create_xdp_rxqs()
1293 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_xdp_tx_frame()  local
1302 	txch = cpsw->txv[0].ch;  in cpsw_xdp_tx_frame()
1326 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_run_xdp()  local
1381 	page_pool_recycle_direct(cpsw->page_pool[ch], page);  in cpsw_run_xdp()
1419 	port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_clsflower_add_policer()
1423 		ret = cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id, rate_pkt_ps);  in cpsw_qos_clsflower_add_policer()
1431 		ret = cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, rate_pkt_ps);  in cpsw_qos_clsflower_add_policer()
1505 	u32 port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_delete_clsflower()
1510 		cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id, 0);  in cpsw_qos_delete_clsflower()
1516 		cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, 0);  in cpsw_qos_delete_clsflower()
1573 	u32 port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_clsflower_resume()
1576 		cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id,  in cpsw_qos_clsflower_resume()
1580 		cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id,  in cpsw_qos_clsflower_resume()