1 // SPDX-License-Identifier: GPL-2.0-only
3  * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips
113 	 * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than
114 	 * 64K and a 32-bit bitfield over 2K blocks covers it.
156 		sram = &ad->txcache;  in admac_alloc_sram_carveout()
158 		sram = &ad->rxcache;  in admac_alloc_sram_carveout()
160 	mutex_lock(&ad->cache_alloc_lock);  in admac_alloc_sram_carveout()
162 	nblocks = sram->size / SRAM_BLOCK;  in admac_alloc_sram_carveout()
164 		if (!(sram->allocated & BIT(i)))  in admac_alloc_sram_carveout()
170 		sram->allocated |= BIT(i);  in admac_alloc_sram_carveout()
172 		ret = -EBUSY;  in admac_alloc_sram_carveout()
175 	mutex_unlock(&ad->cache_alloc_lock);  in admac_alloc_sram_carveout()
189 		sram = &ad->txcache;  in admac_free_sram_carveout()
191 		sram = &ad->rxcache;  in admac_free_sram_carveout()
193 	if (WARN_ON(base >= sram->size))  in admac_free_sram_carveout()
196 	mutex_lock(&ad->cache_alloc_lock);  in admac_free_sram_carveout()
198 	sram->allocated &= ~BIT(i);  in admac_free_sram_carveout()
199 	mutex_unlock(&ad->cache_alloc_lock);  in admac_free_sram_carveout()
204 	void __iomem *addr = ad->base + reg;  in admac_modify()
229 	struct admac_chan *adchan = to_admac_chan(tx->chan);  in admac_tx_submit()
233 	spin_lock_irqsave(&adchan->lock, flags);  in admac_tx_submit()
235 	list_add_tail(&adtx->node, &adchan->submitted);  in admac_tx_submit()
236 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_tx_submit()
256 	if (direction != admac_chan_direction(adchan->no))  in admac_prep_dma_cyclic()
263 	adtx->cyclic = true;  in admac_prep_dma_cyclic()
265 	adtx->buf_addr = buf_addr;  in admac_prep_dma_cyclic()
266 	adtx->buf_len = buf_len;  in admac_prep_dma_cyclic()
267 	adtx->buf_end = buf_addr + buf_len;  in admac_prep_dma_cyclic()
268 	adtx->period_len = period_len;  in admac_prep_dma_cyclic()
270 	adtx->submitted_pos = 0;  in admac_prep_dma_cyclic()
271 	adtx->reclaimed_pos = 0;  in admac_prep_dma_cyclic()
273 	dma_async_tx_descriptor_init(&adtx->tx, chan);  in admac_prep_dma_cyclic()
274 	adtx->tx.tx_submit = admac_tx_submit;  in admac_prep_dma_cyclic()
275 	adtx->tx.desc_free = admac_desc_free;  in admac_prep_dma_cyclic()
277 	return &adtx->tx;  in admac_prep_dma_cyclic()
288 	addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len);  in admac_cyclic_write_one_desc()
291 	WARN_ON_ONCE(addr + tx->period_len > tx->buf_end);  in admac_cyclic_write_one_desc()
293 	dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n",  in admac_cyclic_write_one_desc()
294 		channo, &addr, tx->period_len, FLAG_DESC_NOTIFY);  in admac_cyclic_write_one_desc()
296 	writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
297 	writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
298 	writel_relaxed(tx->period_len,      ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
299 	writel_relaxed(FLAG_DESC_NOTIFY,    ad->base + REG_DESC_WRITE(channo));  in admac_cyclic_write_one_desc()
301 	tx->submitted_pos += tx->period_len;  in admac_cyclic_write_one_desc()
302 	tx->submitted_pos %= 2 * tx->buf_len;  in admac_cyclic_write_one_desc()
315 		if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL)  in admac_cyclic_write_desc()
327 		return (wrslot + 4 - rdslot) % 4;  in admac_ring_noccupied_slots()
349 	ring1 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));  in admac_cyclic_read_residue()
350 	residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo));  in admac_cyclic_read_residue()
351 	ring2 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));  in admac_cyclic_read_residue()
352 	residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo));  in admac_cyclic_read_residue()
365 	pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2;  in admac_cyclic_read_residue()
367 	return adtx->buf_len - pos % adtx->buf_len;  in admac_cyclic_read_residue()
374 	struct admac_data *ad = adchan->host;  in admac_tx_status()
385 	spin_lock_irqsave(&adchan->lock, flags);  in admac_tx_status()
386 	adtx = adchan->current_tx;  in admac_tx_status()
388 	if (adtx && adtx->tx.cookie == cookie) {  in admac_tx_status()
390 		residue = admac_cyclic_read_residue(ad, adchan->no, adtx);  in admac_tx_status()
394 		list_for_each_entry(adtx, &adchan->issued, node) {  in admac_tx_status()
395 			if (adtx->tx.cookie == cookie) {  in admac_tx_status()
396 				residue = adtx->buf_len;  in admac_tx_status()
401 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_tx_status()
409 	struct admac_data *ad = adchan->host;  in admac_start_chan()
410 	u32 startbit = 1 << (adchan->no / 2);  in admac_start_chan()
413 		       ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index));  in admac_start_chan()
415 		       ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index));  in admac_start_chan()
417 	switch (admac_chan_direction(adchan->no)) {  in admac_start_chan()
419 		writel_relaxed(startbit, ad->base + REG_TX_START);  in admac_start_chan()
422 		writel_relaxed(startbit, ad->base + REG_RX_START);  in admac_start_chan()
427 	dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no);  in admac_start_chan()
432 	struct admac_data *ad = adchan->host;  in admac_stop_chan()
433 	u32 stopbit = 1 << (adchan->no / 2);  in admac_stop_chan()
435 	switch (admac_chan_direction(adchan->no)) {  in admac_stop_chan()
437 		writel_relaxed(stopbit, ad->base + REG_TX_STOP);  in admac_stop_chan()
440 		writel_relaxed(stopbit, ad->base + REG_RX_STOP);  in admac_stop_chan()
445 	dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no);  in admac_stop_chan()
450 	struct admac_data *ad = adchan->host;  in admac_reset_rings()
453 		       ad->base + REG_CHAN_CTL(adchan->no));  in admac_reset_rings()
454 	writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));  in admac_reset_rings()
459 	struct admac_data *ad = adchan->host;  in admac_start_current_tx()
460 	int ch = adchan->no;  in admac_start_current_tx()
463 	writel_relaxed(0, ad->base + REG_CHAN_CTL(ch));  in admac_start_current_tx()
465 	admac_cyclic_write_one_desc(ad, ch, adchan->current_tx);  in admac_start_current_tx()
467 	admac_cyclic_write_desc(ad, ch, adchan->current_tx);  in admac_start_current_tx()
476 	spin_lock_irqsave(&adchan->lock, flags);  in admac_issue_pending()
477 	list_splice_tail_init(&adchan->submitted, &adchan->issued);  in admac_issue_pending()
478 	if (!list_empty(&adchan->issued) && !adchan->current_tx) {  in admac_issue_pending()
479 		tx = list_first_entry(&adchan->issued, struct admac_tx, node);  in admac_issue_pending()
480 		list_del(&tx->node);  in admac_issue_pending()
482 		adchan->current_tx = tx;  in admac_issue_pending()
483 		adchan->nperiod_acks = 0;  in admac_issue_pending()
486 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_issue_pending()
512 	spin_lock_irqsave(&adchan->lock, flags);  in admac_terminate_all()
516 	if (adchan->current_tx) {  in admac_terminate_all()
517 		list_add_tail(&adchan->current_tx->node, &adchan->to_free);  in admac_terminate_all()
518 		adchan->current_tx = NULL;  in admac_terminate_all()
524 	list_splice_tail_init(&adchan->submitted, &adchan->to_free);  in admac_terminate_all()
525 	list_splice_tail_init(&adchan->issued, &adchan->to_free);  in admac_terminate_all()
526 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_terminate_all()
538 	spin_lock_irqsave(&adchan->lock, flags);  in admac_synchronize()
539 	list_splice_tail_init(&adchan->to_free, &head);  in admac_synchronize()
540 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_synchronize()
542 	tasklet_kill(&adchan->tasklet);  in admac_synchronize()
545 		list_del(&adtx->node);  in admac_synchronize()
546 		admac_desc_free(&adtx->tx);  in admac_synchronize()
553 	struct admac_data *ad = adchan->host;  in admac_alloc_chan_resources()
556 	dma_cookie_init(&adchan->chan);  in admac_alloc_chan_resources()
557 	ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no),  in admac_alloc_chan_resources()
558 					&adchan->carveout);  in admac_alloc_chan_resources()
562 	writel_relaxed(adchan->carveout,  in admac_alloc_chan_resources()
563 		       ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no));  in admac_alloc_chan_resources()
573 	admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no),  in admac_free_chan_resources()
574 				 adchan->carveout);  in admac_free_chan_resources()
580 	struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data;  in admac_dma_of_xlate()
583 	if (dma_spec->args_count != 1)  in admac_dma_of_xlate()
586 	index = dma_spec->args[0];  in admac_dma_of_xlate()
588 	if (index >= ad->nchannels) {  in admac_dma_of_xlate()
589 		dev_err(ad->dev, "channel index %u out of bounds\n", index);  in admac_dma_of_xlate()
593 	return dma_get_slave_channel(&ad->channels[index].chan);  in admac_dma_of_xlate()
603 		if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY)  in admac_drain_reports()
606 		countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
607 		countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
608 		unk1 =        readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
609 		flags =       readl_relaxed(ad->base + REG_REPORT_READ(channo));  in admac_drain_reports()
611 		dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n",  in admac_drain_reports()
622 	if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) {  in admac_handle_status_err()
623 		writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo));  in admac_handle_status_err()
624 		dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo);  in admac_handle_status_err()
628 	if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) {  in admac_handle_status_err()
629 		writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo));  in admac_handle_status_err()
630 		dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo);  in admac_handle_status_err()
635 		dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo);  in admac_handle_status_err()
636 		admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index),  in admac_handle_status_err()
643 	struct admac_chan *adchan = &ad->channels[channo];  in admac_handle_status_desc_done()
648 		       ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index));  in admac_handle_status_desc_done()
650 	spin_lock_irqsave(&adchan->lock, flags);  in admac_handle_status_desc_done()
653 	if (adchan->current_tx) {  in admac_handle_status_desc_done()
654 		struct admac_tx *tx = adchan->current_tx;  in admac_handle_status_desc_done()
656 		adchan->nperiod_acks += nreports;  in admac_handle_status_desc_done()
657 		tx->reclaimed_pos += nreports * tx->period_len;  in admac_handle_status_desc_done()
658 		tx->reclaimed_pos %= 2 * tx->buf_len;  in admac_handle_status_desc_done()
661 		tasklet_schedule(&adchan->tasklet);  in admac_handle_status_desc_done()
663 	spin_unlock_irqrestore(&adchan->lock, flags);  in admac_handle_status_desc_done()
668 	u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index));  in admac_handle_chan_int()
683 	rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index));  in admac_interrupt()
684 	tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index));  in admac_interrupt()
685 	global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));  in admac_interrupt()
690 	for (i = 0; i < ad->nchannels; i += 2) {  in admac_interrupt()
696 	for (i = 1; i < ad->nchannels; i += 2) {  in admac_interrupt()
703 		dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n",  in admac_interrupt()
705 		writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index));  in admac_interrupt()
719 	spin_lock_irq(&adchan->lock);  in admac_chan_tasklet()
720 	adtx = adchan->current_tx;  in admac_chan_tasklet()
721 	nacks = adchan->nperiod_acks;  in admac_chan_tasklet()
722 	adchan->nperiod_acks = 0;  in admac_chan_tasklet()
723 	spin_unlock_irq(&adchan->lock);  in admac_chan_tasklet()
731 	dmaengine_desc_get_callback(&adtx->tx, &cb);  in admac_chan_tasklet()
732 	while (nacks--)  in admac_chan_tasklet()
740 	struct admac_data *ad = adchan->host;  in admac_device_config()
741 	bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV;  in admac_device_config()
745 	switch (is_tx ? config->dst_addr_width : config->src_addr_width) {  in admac_device_config()
759 		return -EINVAL;  in admac_device_config()
765 	 * The controller has some means of out-of-band signalling, to the peripheral,  in admac_device_config()
769 	switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) {  in admac_device_config()
779 		return -EINVAL;  in admac_device_config()
782 	writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no));  in admac_device_config()
786 	 * held in controller's per-channel FIFO. Transfers seem to be triggered  in admac_device_config()
793 		       ad->base + REG_CHAN_FIFOCTL(adchan->no));  in admac_device_config()
800 	struct device_node *np = pdev->dev.of_node;  in admac_probe()
806 	err = of_property_read_u32(np, "dma-channels", &nchannels);  in admac_probe()
808 		dev_err(&pdev->dev, "missing or invalid dma-channels property\n");  in admac_probe()
809 		return -EINVAL;  in admac_probe()
812 	ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL);  in admac_probe()
814 		return -ENOMEM;  in admac_probe()
817 	ad->dev = &pdev->dev;  in admac_probe()
818 	ad->nchannels = nchannels;  in admac_probe()
819 	mutex_init(&ad->cache_alloc_lock);  in admac_probe()
828 			ad->irq_index = i;  in admac_probe()
834 		return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n");  in admac_probe()
835 	ad->irq = irq;  in admac_probe()
837 	ad->base = devm_platform_ioremap_resource(pdev, 0);  in admac_probe()
838 	if (IS_ERR(ad->base))  in admac_probe()
839 		return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),  in admac_probe()
842 	ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);  in admac_probe()
843 	if (IS_ERR(ad->rstc))  in admac_probe()
844 		return PTR_ERR(ad->rstc);  in admac_probe()
846 	dma = &ad->dma;  in admac_probe()
848 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);  in admac_probe()
849 	dma_cap_set(DMA_CYCLIC, dma->cap_mask);  in admac_probe()
851 	dma->dev = &pdev->dev;  in admac_probe()
852 	dma->device_alloc_chan_resources = admac_alloc_chan_resources;  in admac_probe()
853 	dma->device_free_chan_resources = admac_free_chan_resources;  in admac_probe()
854 	dma->device_tx_status = admac_tx_status;  in admac_probe()
855 	dma->device_issue_pending = admac_issue_pending;  in admac_probe()
856 	dma->device_terminate_all = admac_terminate_all;  in admac_probe()
857 	dma->device_synchronize = admac_synchronize;  in admac_probe()
858 	dma->device_prep_dma_cyclic = admac_prep_dma_cyclic;  in admac_probe()
859 	dma->device_config = admac_device_config;  in admac_probe()
860 	dma->device_pause = admac_pause;  in admac_probe()
861 	dma->device_resume = admac_resume;  in admac_probe()
863 	dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);  in admac_probe()
864 	dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in admac_probe()
865 	dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |  in admac_probe()
868 	dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |  in admac_probe()
872 	INIT_LIST_HEAD(&dma->channels);  in admac_probe()
874 		struct admac_chan *adchan = &ad->channels[i];  in admac_probe()
876 		adchan->host = ad;  in admac_probe()
877 		adchan->no = i;  in admac_probe()
878 		adchan->chan.device = &ad->dma;  in admac_probe()
879 		spin_lock_init(&adchan->lock);  in admac_probe()
880 		INIT_LIST_HEAD(&adchan->submitted);  in admac_probe()
881 		INIT_LIST_HEAD(&adchan->issued);  in admac_probe()
882 		INIT_LIST_HEAD(&adchan->to_free);  in admac_probe()
883 		list_add_tail(&adchan->chan.device_node, &dma->channels);  in admac_probe()
884 		tasklet_setup(&adchan->tasklet, admac_chan_tasklet);  in admac_probe()
887 	err = reset_control_reset(ad->rstc);  in admac_probe()
889 		return dev_err_probe(&pdev->dev, err,  in admac_probe()
892 	err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad);  in admac_probe()
894 		dev_err_probe(&pdev->dev, err,  in admac_probe()
899 	err = dma_async_device_register(&ad->dma);  in admac_probe()
901 		dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");  in admac_probe()
905 	err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);  in admac_probe()
907 		dma_async_device_unregister(&ad->dma);  in admac_probe()
908 		dev_err_probe(&pdev->dev, err, "failed to register with OF\n");  in admac_probe()
912 	ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE);  in admac_probe()
913 	ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE);  in admac_probe()
915 	dev_info(&pdev->dev, "Audio DMA Controller\n");  in admac_probe()
916 	dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n",  in admac_probe()
917 		 readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size);  in admac_probe()
922 	free_irq(ad->irq, ad);  in admac_probe()
924 	reset_control_rearm(ad->rstc);  in admac_probe()
932 	of_dma_controller_free(pdev->dev.of_node);  in admac_remove()
933 	dma_async_device_unregister(&ad->dma);  in admac_remove()
934 	free_irq(ad->irq, ad);  in admac_remove()
935 	reset_control_rearm(ad->rstc);  in admac_remove()
941 	{ .compatible = "apple,admac", },
948 		.name = "apple-admac",
957 MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs");