148 static int admac_alloc_sram_carveout(struct admac_data *ad,  in admac_alloc_sram_carveout()  argument
156 		sram = &ad->txcache;  in admac_alloc_sram_carveout()
158 		sram = &ad->rxcache;  in admac_alloc_sram_carveout()
160 	mutex_lock(&ad->cache_alloc_lock);  in admac_alloc_sram_carveout()
175 	mutex_unlock(&ad->cache_alloc_lock);  in admac_alloc_sram_carveout()
180 static void admac_free_sram_carveout(struct admac_data *ad,  in admac_free_sram_carveout()  argument
189 		sram = &ad->txcache;  in admac_free_sram_carveout()
191 		sram = &ad->rxcache;  in admac_free_sram_carveout()
196 	mutex_lock(&ad->cache_alloc_lock);  in admac_free_sram_carveout()
199 	mutex_unlock(&ad->cache_alloc_lock);  in admac_free_sram_carveout()
202 static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val)  in admac_modify()  argument
204 	void __iomem *addr = ad->base + reg;  in admac_modify()
283 static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo,  in admac_cyclic_write_one_desc()  argument
293 	dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n",  in admac_cyclic_write_one_desc()
296 	writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
297 	writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
298 	writel_relaxed(tx->period_len,      ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
299 	writel_relaxed(FLAG_DESC_NOTIFY,    ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
309 static void admac_cyclic_write_desc(struct admac_data *ad, int channo,  in admac_cyclic_write_desc()  argument
315 		if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL)  in admac_cyclic_write_desc()
317 		admac_cyclic_write_one_desc(ad, channo, tx);  in admac_cyclic_write_desc()
341 static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo,  in admac_cyclic_read_residue()  argument
349 	ring1 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));  in admac_cyclic_read_residue()
350 	residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo));  in admac_cyclic_read_residue()
351 	ring2 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));  in admac_cyclic_read_residue()
352 	residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo));  in admac_cyclic_read_residue()
374 	struct admac_data *ad = adchan->host;  in admac_tx_status()  local
390 		residue = admac_cyclic_read_residue(ad, adchan->no, adtx);  in admac_tx_status()
409 	struct admac_data *ad = adchan->host;  in admac_start_chan()  local
413 		       ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index));  in admac_start_chan()
415 		       ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index));  in admac_start_chan()
419 		writel_relaxed(startbit, ad->base + REG_TX_START);  in admac_start_chan()
422 		writel_relaxed(startbit, ad->base + REG_RX_START);  in admac_start_chan()
432 	struct admac_data *ad = adchan->host;  in admac_stop_chan()  local
437 		writel_relaxed(stopbit, ad->base + REG_TX_STOP);  in admac_stop_chan()
440 		writel_relaxed(stopbit, ad->base + REG_RX_STOP);  in admac_stop_chan()
450 	struct admac_data *ad = adchan->host;  in admac_reset_rings()  local
453 		       ad->base + REG_CHAN_CTL(adchan->no));  in admac_reset_rings()
454 	writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));  in admac_reset_rings()
459 	struct admac_data *ad = adchan->host;  in admac_start_current_tx()  local
463 	writel_relaxed(0, ad->base + REG_CHAN_CTL(ch));  in admac_start_current_tx()
465 	admac_cyclic_write_one_desc(ad, ch, adchan->current_tx);  in admac_start_current_tx()
467 	admac_cyclic_write_desc(ad, ch, adchan->current_tx);  in admac_start_current_tx()
553 	struct admac_data *ad = adchan->host;  in admac_alloc_chan_resources()  local
557 	ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no),  in admac_alloc_chan_resources()
563 		       ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no));  in admac_alloc_chan_resources()
580 	struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data;  in admac_dma_of_xlate()  local
588 	if (index >= ad->nchannels) {  in admac_dma_of_xlate()
589 		dev_err(ad->dev, "channel index %u out of bounds\n", index);  in admac_dma_of_xlate()
593 	return dma_get_slave_channel(&ad->channels[index].chan);  in admac_dma_of_xlate()
596 static int admac_drain_reports(struct admac_data *ad, int channo)  in admac_drain_reports()  argument
603 		if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY)  in admac_drain_reports()
606 		countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
607 		countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
608 		unk1 =        readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
609 		flags =       readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
611 		dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n",  in admac_drain_reports()
618 static void admac_handle_status_err(struct admac_data *ad, int channo)  in admac_handle_status_err()  argument
622 	if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) {  in admac_handle_status_err()
623 		writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo));  in admac_handle_status_err()
624 		dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo);  in admac_handle_status_err()
628 	if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) {  in admac_handle_status_err()
629 		writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo));  in admac_handle_status_err()
630 		dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo);  in admac_handle_status_err()
635 		dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo);  in admac_handle_status_err()
636 		admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index),  in admac_handle_status_err()
641 static void admac_handle_status_desc_done(struct admac_data *ad, int channo)  in admac_handle_status_desc_done()  argument
643 	struct admac_chan *adchan = &ad->channels[channo];  in admac_handle_status_desc_done()
648 		       ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index));  in admac_handle_status_desc_done()
651 	nreports = admac_drain_reports(ad, channo);  in admac_handle_status_desc_done()
660 		admac_cyclic_write_desc(ad, channo, tx);  in admac_handle_status_desc_done()
666 static void admac_handle_chan_int(struct admac_data *ad, int no)  in admac_handle_chan_int()  argument
668 	u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index));  in admac_handle_chan_int()
671 		admac_handle_status_err(ad, no);  in admac_handle_chan_int()
674 		admac_handle_status_desc_done(ad, no);  in admac_handle_chan_int()
679 	struct admac_data *ad = devid;  in admac_interrupt()  local
683 	rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index));  in admac_interrupt()
684 	tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index));  in admac_interrupt()
685 	global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));  in admac_interrupt()
690 	for (i = 0; i < ad->nchannels; i += 2) {  in admac_interrupt()
692 			admac_handle_chan_int(ad, i);  in admac_interrupt()
696 	for (i = 1; i < ad->nchannels; i += 2) {  in admac_interrupt()
698 			admac_handle_chan_int(ad, i);  in admac_interrupt()
703 		dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n",  in admac_interrupt()
705 		writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));  in admac_interrupt()
740 	struct admac_data *ad = adchan->host;  in admac_device_config()  local
782 	writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no));  in admac_device_config()
793 		       ad->base + REG_CHAN_FIFOCTL(adchan->no));  in admac_device_config()
801 	struct admac_data *ad;  in admac_probe()  local
812 	ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL);  in admac_probe()
813 	if (!ad)  in admac_probe()
816 	platform_set_drvdata(pdev, ad);  in admac_probe()
817 	ad->dev = &pdev->dev;  in admac_probe()
818 	ad->nchannels = nchannels;  in admac_probe()
819 	mutex_init(&ad->cache_alloc_lock);  in admac_probe()
828 			ad->irq_index = i;  in admac_probe()
835 	ad->irq = irq;  in admac_probe()
837 	ad->base = devm_platform_ioremap_resource(pdev, 0);  in admac_probe()
838 	if (IS_ERR(ad->base))  in admac_probe()
839 		return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),  in admac_probe()
842 	ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);  in admac_probe()
843 	if (IS_ERR(ad->rstc))  in admac_probe()
844 		return PTR_ERR(ad->rstc);  in admac_probe()
846 	dma = &ad->dma;  in admac_probe()
874 		struct admac_chan *adchan = &ad->channels[i];  in admac_probe()
876 		adchan->host = ad;  in admac_probe()
878 		adchan->chan.device = &ad->dma;  in admac_probe()
887 	err = reset_control_reset(ad->rstc);  in admac_probe()
892 	err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad);  in admac_probe()
899 	err = dma_async_device_register(&ad->dma);  in admac_probe()
905 	err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);  in admac_probe()
907 		dma_async_device_unregister(&ad->dma);  in admac_probe()
912 	ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE);  in admac_probe()
913 	ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE);  in admac_probe()
917 		 readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size);  in admac_probe()
922 	free_irq(ad->irq, ad);  in admac_probe()
924 	reset_control_rearm(ad->rstc);  in admac_probe()
930 	struct admac_data *ad = platform_get_drvdata(pdev);  in admac_remove()  local
933 	dma_async_device_unregister(&ad->dma);  in admac_remove()
934 	free_irq(ad->irq, ad);  in admac_remove()
935 	reset_control_rearm(ad->rstc);  in admac_remove()