96 static void cio2_fbpt_exit_dummy(struct cio2_device *cio2)  in cio2_fbpt_exit_dummy()  argument
98 	if (cio2->dummy_lop) {  in cio2_fbpt_exit_dummy()
99 		dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,  in cio2_fbpt_exit_dummy()
100 				  cio2->dummy_lop, cio2->dummy_lop_bus_addr);  in cio2_fbpt_exit_dummy()
101 		cio2->dummy_lop = NULL;  in cio2_fbpt_exit_dummy()
103 	if (cio2->dummy_page) {  in cio2_fbpt_exit_dummy()
104 		dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,  in cio2_fbpt_exit_dummy()
105 				  cio2->dummy_page, cio2->dummy_page_bus_addr);  in cio2_fbpt_exit_dummy()
106 		cio2->dummy_page = NULL;  in cio2_fbpt_exit_dummy()
110 static int cio2_fbpt_init_dummy(struct cio2_device *cio2)  in cio2_fbpt_init_dummy()  argument
114 	cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev,  in cio2_fbpt_init_dummy()
116 					      &cio2->dummy_page_bus_addr,  in cio2_fbpt_init_dummy()
118 	cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev,  in cio2_fbpt_init_dummy()
120 					     &cio2->dummy_lop_bus_addr,  in cio2_fbpt_init_dummy()
122 	if (!cio2->dummy_page || !cio2->dummy_lop) {  in cio2_fbpt_init_dummy()
123 		cio2_fbpt_exit_dummy(cio2);  in cio2_fbpt_init_dummy()
130 	for (i = 0; i < CIO2_PAGE_SIZE / sizeof(*cio2->dummy_lop); i++)  in cio2_fbpt_init_dummy()
131 		cio2->dummy_lop[i] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;  in cio2_fbpt_init_dummy()
136 static void cio2_fbpt_entry_enable(struct cio2_device *cio2,  in cio2_fbpt_entry_enable()  argument
156 static void cio2_fbpt_entry_init_dummy(struct cio2_device *cio2,  in cio2_fbpt_entry_init_dummy()  argument
168 		entry[i].lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;  in cio2_fbpt_entry_init_dummy()
170 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_dummy()
174 static void cio2_fbpt_entry_init_buf(struct cio2_device *cio2,  in cio2_fbpt_entry_init_buf()  argument
212 	entry->lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;  in cio2_fbpt_entry_init_buf()
214 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_buf()
217 static int cio2_fbpt_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_init()  argument
219 	struct device *dev = &cio2->pci_dev->dev;  in cio2_fbpt_init()
296 static int cio2_csi2_calc_timing(struct cio2_device *cio2, struct cio2_queue *q,  in cio2_csi2_calc_timing()  argument
299 	struct device *dev = &cio2->pci_dev->dev;  in cio2_csi2_calc_timing()
352 static int cio2_hw_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_init()  argument
361 	void __iomem *const base = cio2->base;  in cio2_hw_init()
373 	r = cio2_csi2_calc_timing(cio2, q, &timing);  in cio2_hw_init()
515 static void cio2_hw_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_exit()  argument
517 	void __iomem *base = cio2->base;  in cio2_hw_exit()
535 		dev_err(&cio2->pci_dev->dev,  in cio2_hw_exit()
546 static void cio2_buffer_done(struct cio2_device *cio2, unsigned int dma_chan)  in cio2_buffer_done()  argument
548 	struct device *dev = &cio2->pci_dev->dev;  in cio2_buffer_done()
549 	struct cio2_queue *q = cio2->cur_queue;  in cio2_buffer_done()
573 			dev_dbg(&cio2->pci_dev->dev,  in cio2_buffer_done()
586 		cio2_fbpt_entry_init_dummy(cio2, entry);  in cio2_buffer_done()
592 		dev_warn(&cio2->pci_dev->dev,  in cio2_buffer_done()
597 static void cio2_queue_event_sof(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_event_sof()  argument
643 static void cio2_irq_handle_once(struct cio2_device *cio2, u32 int_status)  in cio2_irq_handle_once()  argument
645 	void __iomem *const base = cio2->base;  in cio2_irq_handle_once()
646 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq_handle_once()
686 				cio2_buffer_done(cio2, d);  in cio2_irq_handle_once()
700 					cio2_queue_event_sof(cio2,  in cio2_irq_handle_once()
701 							     cio2->cur_queue);  in cio2_irq_handle_once()
771 	struct cio2_device *cio2 = cio2_ptr;  in cio2_irq()  local
772 	void __iomem *const base = cio2->base;  in cio2_irq()
773 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq()
783 		cio2_irq_handle_once(cio2, int_status);  in cio2_irq()
814 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_queue_setup()  local
822 		alloc_devs[i] = &cio2->pci_dev->dev;  in cio2_vb2_queue_setup()
830 		cio2_fbpt_entry_init_dummy(cio2, &q->fbpt[i * CIO2_MAX_LOPS]);  in cio2_vb2_queue_setup()
842 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_init()  local
843 	struct device *dev = &cio2->pci_dev->dev;  in cio2_vb2_buf_init()
889 	b->lop[i][j] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;  in cio2_vb2_buf_init()
901 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_queue()  local
912 	dev_dbg(&cio2->pci_dev->dev, "queue buffer %d\n", vb->index);  in cio2_vb2_buf_queue()
929 	fbpt_rp = (readl(cio2->base + CIO2_REG_CDMARI(CIO2_DMA_CHAN))  in cio2_vb2_buf_queue()
954 			cio2_fbpt_entry_init_buf(cio2, b, entry);  in cio2_vb2_buf_queue()
963 		dev_dbg(&cio2->pci_dev->dev, "entry %i was full!\n", next);  in cio2_vb2_buf_queue()
968 	dev_err(&cio2->pci_dev->dev, "error: all cio2 entries were full!\n");  in cio2_vb2_buf_queue()
976 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_cleanup()  local
984 			dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,  in cio2_vb2_buf_cleanup()
992 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_start_streaming()  local
995 	cio2->cur_queue = q;  in cio2_vb2_start_streaming()
998 	r = pm_runtime_get_sync(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
1000 		dev_info(&cio2->pci_dev->dev, "failed to set power %d\n", r);  in cio2_vb2_start_streaming()
1001 		pm_runtime_put_noidle(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
1009 	r = cio2_hw_init(cio2, q);  in cio2_vb2_start_streaming()
1018 	cio2->streaming = true;  in cio2_vb2_start_streaming()
1023 	cio2_hw_exit(cio2, q);  in cio2_vb2_start_streaming()
1027 	dev_dbg(&cio2->pci_dev->dev, "failed to start streaming (%d)\n", r);  in cio2_vb2_start_streaming()
1029 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
1037 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_stop_streaming()  local
1040 		dev_err(&cio2->pci_dev->dev,  in cio2_vb2_stop_streaming()
1043 	cio2_hw_exit(cio2, q);  in cio2_vb2_stop_streaming()
1044 	synchronize_irq(cio2->pci_dev->irq);  in cio2_vb2_stop_streaming()
1047 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_stop_streaming()
1048 	cio2->streaming = false;  in cio2_vb2_stop_streaming()
1067 	struct cio2_device *cio2 = video_drvdata(file);  in cio2_v4l2_querycap()  local
1072 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_v4l2_querycap()
1339 	struct cio2_device *cio2 = video_get_drvdata(vd);  in cio2_video_link_validate()  local
1344 		dev_info(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1355 		dev_err(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1401 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_bound()  local
1407 	if (cio2->queue[s_asd->csi2.port].sensor)  in cio2_notifier_bound()
1410 	q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_bound()
1414 	q->csi_rx_base = cio2->base + CIO2_REG_PIPE_BASE(q->csi2.port);  in cio2_notifier_bound()
1424 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_unbind()  local
1429 	cio2->queue[s_asd->csi2.port].sensor = NULL;  in cio2_notifier_unbind()
1435 	struct cio2_device *cio2 = container_of(notifier, struct cio2_device,  in cio2_notifier_complete()  local
1443 		s_asd = container_of(cio2->notifier.subdevs[i],  in cio2_notifier_complete()
1445 		q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_complete()
1453 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1464 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1466 				cio2->queue[i].sensor->name);  in cio2_notifier_complete()
1471 	return v4l2_device_register_subdev_nodes(&cio2->v4l2_dev);  in cio2_notifier_complete()
1498 static int cio2_notifier_init(struct cio2_device *cio2)  in cio2_notifier_init()  argument
1503 		&cio2->pci_dev->dev, &cio2->notifier,  in cio2_notifier_init()
1509 	if (!cio2->notifier.num_subdevs)  in cio2_notifier_init()
1512 	cio2->notifier.ops = &cio2_async_ops;  in cio2_notifier_init()
1513 	ret = v4l2_async_notifier_register(&cio2->v4l2_dev, &cio2->notifier);  in cio2_notifier_init()
1515 		dev_err(&cio2->pci_dev->dev,  in cio2_notifier_init()
1517 		v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_notifier_init()
1523 static void cio2_notifier_exit(struct cio2_device *cio2)  in cio2_notifier_exit()  argument
1525 	v4l2_async_notifier_unregister(&cio2->notifier);  in cio2_notifier_exit()
1526 	v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_notifier_exit()
1538 static int cio2_queue_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_init()  argument
1572 	r = cio2_fbpt_init(cio2, q);  in cio2_queue_init()
1584 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1593 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1603 		 CIO2_ENTITY_NAME " %td", q - cio2->queue);  in cio2_queue_init()
1604 	v4l2_set_subdevdata(subdev, cio2);  in cio2_queue_init()
1605 	r = v4l2_device_register_subdev(&cio2->v4l2_dev, subdev);  in cio2_queue_init()
1607 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1620 	vbq->drv_priv = cio2;  in cio2_queue_init()
1624 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1631 		 "%s %td", CIO2_NAME, q - cio2->queue);  in cio2_queue_init()
1635 	vdev->lock = &cio2->lock;  in cio2_queue_init()
1636 	vdev->v4l2_dev = &cio2->v4l2_dev;  in cio2_queue_init()
1639 	video_set_drvdata(vdev, cio2);  in cio2_queue_init()
1642 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1667 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_init()
1674 static void cio2_queue_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_exit()  argument
1681 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_exit()
1685 static int cio2_queues_init(struct cio2_device *cio2)  in cio2_queues_init()  argument
1690 		r = cio2_queue_init(cio2, &cio2->queue[i]);  in cio2_queues_init()
1699 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_init()
1704 static void cio2_queues_exit(struct cio2_device *cio2)  in cio2_queues_exit()  argument
1709 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_exit()
1735 	struct cio2_device *cio2;  in cio2_pci_probe()  local
1739 	cio2 = devm_kzalloc(&pci_dev->dev, sizeof(*cio2), GFP_KERNEL);  in cio2_pci_probe()
1740 	if (!cio2)  in cio2_pci_probe()
1742 	cio2->pci_dev = pci_dev;  in cio2_pci_probe()
1765 	cio2->base = iomap[CIO2_PCI_BAR];  in cio2_pci_probe()
1767 	pci_set_drvdata(pci_dev, cio2);  in cio2_pci_probe()
1781 	r = cio2_fbpt_init_dummy(cio2);  in cio2_pci_probe()
1785 	mutex_init(&cio2->lock);  in cio2_pci_probe()
1787 	cio2->media_dev.dev = &cio2->pci_dev->dev;  in cio2_pci_probe()
1788 	strlcpy(cio2->media_dev.model, CIO2_DEVICE_NAME,  in cio2_pci_probe()
1789 		sizeof(cio2->media_dev.model));  in cio2_pci_probe()
1790 	snprintf(cio2->media_dev.bus_info, sizeof(cio2->media_dev.bus_info),  in cio2_pci_probe()
1791 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_pci_probe()
1792 	cio2->media_dev.hw_revision = 0;  in cio2_pci_probe()
1794 	media_device_init(&cio2->media_dev);  in cio2_pci_probe()
1795 	r = media_device_register(&cio2->media_dev);  in cio2_pci_probe()
1799 	cio2->v4l2_dev.mdev = &cio2->media_dev;  in cio2_pci_probe()
1800 	r = v4l2_device_register(&pci_dev->dev, &cio2->v4l2_dev);  in cio2_pci_probe()
1807 	r = cio2_queues_init(cio2);  in cio2_pci_probe()
1812 	r = cio2_notifier_init(cio2);  in cio2_pci_probe()
1817 			     IRQF_SHARED, CIO2_NAME, cio2);  in cio2_pci_probe()
1829 	cio2_notifier_exit(cio2);  in cio2_pci_probe()
1831 	cio2_queues_exit(cio2);  in cio2_pci_probe()
1833 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_probe()
1835 	media_device_unregister(&cio2->media_dev);  in cio2_pci_probe()
1836 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_probe()
1838 	mutex_destroy(&cio2->lock);  in cio2_pci_probe()
1839 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_probe()
1846 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_pci_remove()  local
1849 	cio2_notifier_exit(cio2);  in cio2_pci_remove()
1850 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_remove()
1852 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_pci_remove()
1853 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_remove()
1854 	media_device_unregister(&cio2->media_dev);  in cio2_pci_remove()
1855 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_remove()
1856 	mutex_destroy(&cio2->lock);  in cio2_pci_remove()
1862 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_suspend()  local
1863 	void __iomem *const base = cio2->base;  in cio2_runtime_suspend()
1880 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_resume()  local
1881 	void __iomem *const base = cio2->base;  in cio2_runtime_resume()
1944 static void cio2_fbpt_rearrange(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_rearrange()  argument
1971 		cio2_fbpt_entry_enable(cio2, q->fbpt + i * CIO2_MAX_LOPS);  in cio2_fbpt_rearrange()
1977 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_suspend()  local
1978 	struct cio2_queue *q = cio2->cur_queue;  in cio2_suspend()
1981 	if (!cio2->streaming)  in cio2_suspend()
1985 	cio2_hw_exit(cio2, q);  in cio2_suspend()
1994 	cio2_fbpt_rearrange(cio2, q);  in cio2_suspend()
2004 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_resume()  local
2006 	struct cio2_queue *q = cio2->cur_queue;  in cio2_resume()
2009 	if (!cio2->streaming)  in cio2_resume()
2012 	r = pm_runtime_force_resume(&cio2->pci_dev->dev);  in cio2_resume()
2014 		dev_err(&cio2->pci_dev->dev,  in cio2_resume()
2019 	r = cio2_hw_init(cio2, q);  in cio2_resume()