97 static void cio2_fbpt_exit_dummy(struct cio2_device *cio2)  in cio2_fbpt_exit_dummy()  argument
99 	if (cio2->dummy_lop) {  in cio2_fbpt_exit_dummy()
100 		dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_exit_dummy()
101 				  cio2->dummy_lop, cio2->dummy_lop_bus_addr);  in cio2_fbpt_exit_dummy()
102 		cio2->dummy_lop = NULL;  in cio2_fbpt_exit_dummy()
104 	if (cio2->dummy_page) {  in cio2_fbpt_exit_dummy()
105 		dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_exit_dummy()
106 				  cio2->dummy_page, cio2->dummy_page_bus_addr);  in cio2_fbpt_exit_dummy()
107 		cio2->dummy_page = NULL;  in cio2_fbpt_exit_dummy()
111 static int cio2_fbpt_init_dummy(struct cio2_device *cio2)  in cio2_fbpt_init_dummy()  argument
115 	cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_init_dummy()
116 					      &cio2->dummy_page_bus_addr,  in cio2_fbpt_init_dummy()
118 	cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_init_dummy()
119 					     &cio2->dummy_lop_bus_addr,  in cio2_fbpt_init_dummy()
121 	if (!cio2->dummy_page || !cio2->dummy_lop) {  in cio2_fbpt_init_dummy()
122 		cio2_fbpt_exit_dummy(cio2);  in cio2_fbpt_init_dummy()
130 		cio2->dummy_lop[i] = PFN_DOWN(cio2->dummy_page_bus_addr);  in cio2_fbpt_init_dummy()
135 static void cio2_fbpt_entry_enable(struct cio2_device *cio2,  in cio2_fbpt_entry_enable()  argument
155 static void cio2_fbpt_entry_init_dummy(struct cio2_device *cio2,  in cio2_fbpt_entry_init_dummy()  argument
166 		entry[i].lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);  in cio2_fbpt_entry_init_dummy()
168 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_dummy()
172 static void cio2_fbpt_entry_init_buf(struct cio2_device *cio2,  in cio2_fbpt_entry_init_buf()  argument
208 	entry->lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);  in cio2_fbpt_entry_init_buf()
210 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_buf()
213 static int cio2_fbpt_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_init()  argument
215 	struct device *dev = &cio2->pci_dev->dev;  in cio2_fbpt_init()
290 static int cio2_csi2_calc_timing(struct cio2_device *cio2, struct cio2_queue *q,  in cio2_csi2_calc_timing()  argument
293 	struct device *dev = &cio2->pci_dev->dev;  in cio2_csi2_calc_timing()
346 static int cio2_hw_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_init()  argument
355 	void __iomem *const base = cio2->base;  in cio2_hw_init()
367 	r = cio2_csi2_calc_timing(cio2, q, &timing);  in cio2_hw_init()
508 static void cio2_hw_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_exit()  argument
510 	void __iomem *const base = cio2->base;  in cio2_hw_exit()
527 		dev_err(&cio2->pci_dev->dev,  in cio2_hw_exit()
538 static void cio2_buffer_done(struct cio2_device *cio2, unsigned int dma_chan)  in cio2_buffer_done()  argument
540 	struct device *dev = &cio2->pci_dev->dev;  in cio2_buffer_done()
541 	struct cio2_queue *q = cio2->cur_queue;  in cio2_buffer_done()
552 		dev_warn(&cio2->pci_dev->dev,  in cio2_buffer_done()
568 			dev_dbg(&cio2->pci_dev->dev,  in cio2_buffer_done()
581 		cio2_fbpt_entry_init_dummy(cio2, entry);  in cio2_buffer_done()
587 static void cio2_queue_event_sof(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_event_sof()  argument
633 static void cio2_irq_handle_once(struct cio2_device *cio2, u32 int_status)  in cio2_irq_handle_once()  argument
635 	void __iomem *const base = cio2->base;  in cio2_irq_handle_once()
636 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq_handle_once()
676 				cio2_buffer_done(cio2, d);  in cio2_irq_handle_once()
690 					cio2_queue_event_sof(cio2,  in cio2_irq_handle_once()
691 							     cio2->cur_queue);  in cio2_irq_handle_once()
761 	struct cio2_device *cio2 = cio2_ptr;  in cio2_irq()  local
762 	void __iomem *const base = cio2->base;  in cio2_irq()
763 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq()
773 		cio2_irq_handle_once(cio2, int_status);  in cio2_irq()
804 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_queue_setup()  local
812 		alloc_devs[i] = &cio2->pci_dev->dev;  in cio2_vb2_queue_setup()
820 		cio2_fbpt_entry_init_dummy(cio2, &q->fbpt[i * CIO2_MAX_LOPS]);  in cio2_vb2_queue_setup()
832 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_init()  local
833 	struct device *dev = &cio2->pci_dev->dev;  in cio2_vb2_buf_init()
877 	b->lop[i][j] = PFN_DOWN(cio2->dummy_page_bus_addr);  in cio2_vb2_buf_init()
888 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_queue()  local
899 	dev_dbg(&cio2->pci_dev->dev, "queue buffer %d\n", vb->index);  in cio2_vb2_buf_queue()
916 	fbpt_rp = (readl(cio2->base + CIO2_REG_CDMARI(CIO2_DMA_CHAN))  in cio2_vb2_buf_queue()
941 			cio2_fbpt_entry_init_buf(cio2, b, entry);  in cio2_vb2_buf_queue()
950 		dev_dbg(&cio2->pci_dev->dev, "entry %i was full!\n", next);  in cio2_vb2_buf_queue()
955 	dev_err(&cio2->pci_dev->dev, "error: all cio2 entries were full!\n");  in cio2_vb2_buf_queue()
963 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_cleanup()  local
971 			dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_vb2_buf_cleanup()
979 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_start_streaming()  local
982 	cio2->cur_queue = q;  in cio2_vb2_start_streaming()
985 	r = pm_runtime_get_sync(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
987 		dev_info(&cio2->pci_dev->dev, "failed to set power %d\n", r);  in cio2_vb2_start_streaming()
988 		pm_runtime_put_noidle(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
996 	r = cio2_hw_init(cio2, q);  in cio2_vb2_start_streaming()
1005 	cio2->streaming = true;  in cio2_vb2_start_streaming()
1010 	cio2_hw_exit(cio2, q);  in cio2_vb2_start_streaming()
1014 	dev_dbg(&cio2->pci_dev->dev, "failed to start streaming (%d)\n", r);  in cio2_vb2_start_streaming()
1016 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
1024 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_stop_streaming()  local
1027 		dev_err(&cio2->pci_dev->dev,  in cio2_vb2_stop_streaming()
1030 	cio2_hw_exit(cio2, q);  in cio2_vb2_stop_streaming()
1031 	synchronize_irq(cio2->pci_dev->irq);  in cio2_vb2_stop_streaming()
1034 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_stop_streaming()
1035 	cio2->streaming = false;  in cio2_vb2_stop_streaming()
1054 	struct cio2_device *cio2 = video_drvdata(file);  in cio2_v4l2_querycap()  local
1059 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_v4l2_querycap()
1326 	struct cio2_device *cio2 = video_get_drvdata(vd);  in cio2_video_link_validate()  local
1331 		dev_info(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1342 		dev_err(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1388 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_bound()  local
1394 	if (cio2->queue[s_asd->csi2.port].sensor)  in cio2_notifier_bound()
1397 	q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_bound()
1401 	q->csi_rx_base = cio2->base + CIO2_REG_PIPE_BASE(q->csi2.port);  in cio2_notifier_bound()
1411 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_unbind()  local
1416 	cio2->queue[s_asd->csi2.port].sensor = NULL;  in cio2_notifier_unbind()
1422 	struct cio2_device *cio2 = container_of(notifier, struct cio2_device,  in cio2_notifier_complete()  local
1430 	list_for_each_entry(asd, &cio2->notifier.asd_list, asd_list) {  in cio2_notifier_complete()
1432 		q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_complete()
1440 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1451 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1458 	return v4l2_device_register_subdev_nodes(&cio2->v4l2_dev);  in cio2_notifier_complete()
1467 static int cio2_parse_firmware(struct cio2_device *cio2)  in cio2_parse_firmware()  argument
1480 			dev_fwnode(&cio2->pci_dev->dev), i, 0,  in cio2_parse_firmware()
1500 			&cio2->notifier, ep, &s_asd->asd);  in cio2_parse_firmware()
1518 	cio2->notifier.ops = &cio2_async_ops;  in cio2_parse_firmware()
1519 	ret = v4l2_async_notifier_register(&cio2->v4l2_dev, &cio2->notifier);  in cio2_parse_firmware()
1521 		dev_err(&cio2->pci_dev->dev,  in cio2_parse_firmware()
1536 static int cio2_queue_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_init()  argument
1570 	r = cio2_fbpt_init(cio2, q);  in cio2_queue_init()
1582 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1591 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1601 		 CIO2_ENTITY_NAME " %td", q - cio2->queue);  in cio2_queue_init()
1603 	v4l2_set_subdevdata(subdev, cio2);  in cio2_queue_init()
1604 	r = v4l2_device_register_subdev(&cio2->v4l2_dev, subdev);  in cio2_queue_init()
1606 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1619 	vbq->drv_priv = cio2;  in cio2_queue_init()
1623 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1630 		 "%s %td", CIO2_NAME, q - cio2->queue);  in cio2_queue_init()
1634 	vdev->lock = &cio2->lock;  in cio2_queue_init()
1635 	vdev->v4l2_dev = &cio2->v4l2_dev;  in cio2_queue_init()
1638 	video_set_drvdata(vdev, cio2);  in cio2_queue_init()
1641 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1664 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_init()
1671 static void cio2_queue_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_exit()  argument
1677 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_exit()
1681 static int cio2_queues_init(struct cio2_device *cio2)  in cio2_queues_init()  argument
1686 		r = cio2_queue_init(cio2, &cio2->queue[i]);  in cio2_queues_init()
1695 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_init()
1700 static void cio2_queues_exit(struct cio2_device *cio2)  in cio2_queues_exit()  argument
1705 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_exit()
1713 	struct cio2_device *cio2;  in cio2_pci_probe()  local
1716 	cio2 = devm_kzalloc(&pci_dev->dev, sizeof(*cio2), GFP_KERNEL);  in cio2_pci_probe()
1717 	if (!cio2)  in cio2_pci_probe()
1719 	cio2->pci_dev = pci_dev;  in cio2_pci_probe()
1736 	cio2->base = pcim_iomap_table(pci_dev)[CIO2_PCI_BAR];  in cio2_pci_probe()
1738 	pci_set_drvdata(pci_dev, cio2);  in cio2_pci_probe()
1754 	r = cio2_fbpt_init_dummy(cio2);  in cio2_pci_probe()
1758 	mutex_init(&cio2->lock);  in cio2_pci_probe()
1760 	cio2->media_dev.dev = &cio2->pci_dev->dev;  in cio2_pci_probe()
1761 	strscpy(cio2->media_dev.model, CIO2_DEVICE_NAME,  in cio2_pci_probe()
1762 		sizeof(cio2->media_dev.model));  in cio2_pci_probe()
1763 	snprintf(cio2->media_dev.bus_info, sizeof(cio2->media_dev.bus_info),  in cio2_pci_probe()
1764 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_pci_probe()
1765 	cio2->media_dev.hw_revision = 0;  in cio2_pci_probe()
1767 	media_device_init(&cio2->media_dev);  in cio2_pci_probe()
1768 	r = media_device_register(&cio2->media_dev);  in cio2_pci_probe()
1772 	cio2->v4l2_dev.mdev = &cio2->media_dev;  in cio2_pci_probe()
1773 	r = v4l2_device_register(&pci_dev->dev, &cio2->v4l2_dev);  in cio2_pci_probe()
1780 	r = cio2_queues_init(cio2);  in cio2_pci_probe()
1784 	v4l2_async_notifier_init(&cio2->notifier);  in cio2_pci_probe()
1787 	r = cio2_parse_firmware(cio2);  in cio2_pci_probe()
1792 			     IRQF_SHARED, CIO2_NAME, cio2);  in cio2_pci_probe()
1804 	v4l2_async_notifier_unregister(&cio2->notifier);  in cio2_pci_probe()
1805 	v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_pci_probe()
1806 	cio2_queues_exit(cio2);  in cio2_pci_probe()
1808 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_probe()
1810 	media_device_unregister(&cio2->media_dev);  in cio2_pci_probe()
1811 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_probe()
1813 	mutex_destroy(&cio2->lock);  in cio2_pci_probe()
1814 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_probe()
1821 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_pci_remove()  local
1823 	media_device_unregister(&cio2->media_dev);  in cio2_pci_remove()
1824 	v4l2_async_notifier_unregister(&cio2->notifier);  in cio2_pci_remove()
1825 	v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_pci_remove()
1826 	cio2_queues_exit(cio2);  in cio2_pci_remove()
1827 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_remove()
1828 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_remove()
1829 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_remove()
1830 	mutex_destroy(&cio2->lock);  in cio2_pci_remove()
1836 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_suspend()  local
1837 	void __iomem *const base = cio2->base;  in cio2_runtime_suspend()
1854 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_resume()  local
1855 	void __iomem *const base = cio2->base;  in cio2_runtime_resume()
1918 static void cio2_fbpt_rearrange(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_rearrange()  argument
1945 		cio2_fbpt_entry_enable(cio2, q->fbpt + i * CIO2_MAX_LOPS);  in cio2_fbpt_rearrange()
1951 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_suspend()  local
1952 	struct cio2_queue *q = cio2->cur_queue;  in cio2_suspend()
1955 	if (!cio2->streaming)  in cio2_suspend()
1959 	cio2_hw_exit(cio2, q);  in cio2_suspend()
1968 	cio2_fbpt_rearrange(cio2, q);  in cio2_suspend()
1977 	struct cio2_device *cio2 = dev_get_drvdata(dev);  in cio2_resume()  local
1978 	struct cio2_queue *q = cio2->cur_queue;  in cio2_resume()
1982 	if (!cio2->streaming)  in cio2_resume()
1985 	r = pm_runtime_force_resume(&cio2->pci_dev->dev);  in cio2_resume()
1987 		dev_err(&cio2->pci_dev->dev,  in cio2_resume()
1992 	r = cio2_hw_init(cio2, q);  in cio2_resume()