165 static int clone_range_of_sgt_for_slice(struct qaic_device *qdev, struct sg_table **sgt_out,  in clone_range_of_sgt_for_slice()  argument
245 static int encode_reqs(struct qaic_device *qdev, struct bo_slice *slice,  in encode_reqs()  argument
380 static int qaic_map_one_slice(struct qaic_device *qdev, struct qaic_bo *bo,  in qaic_map_one_slice()  argument
387 	ret = clone_range_of_sgt_for_slice(qdev, &sgt, bo->sgt, slice_ent->size, slice_ent->offset);  in qaic_map_one_slice()
411 	ret = encode_reqs(qdev, slice, slice_ent);  in qaic_map_one_slice()
433 static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 size)  in create_sgt()  argument
547 static int qaic_validate_req(struct qaic_device *qdev, struct qaic_attach_slice_entry *slice_ent,  in qaic_validate_req()  argument
657 	struct qaic_device *qdev;  in qaic_create_bo_ioctl()  local
677 	qdev = usr->qddev->qdev;  in qaic_create_bo_ioctl()
678 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in qaic_create_bo_ioctl()
679 	if (qdev->in_reset) {  in qaic_create_bo_ioctl()
694 	ret = create_sgt(qdev, &bo->sgt, size);  in qaic_create_bo_ioctl()
706 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_create_bo_ioctl()
716 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_create_bo_ioctl()
727 	struct qaic_device *qdev;  in qaic_mmap_bo_ioctl()  local
738 	qdev = usr->qddev->qdev;  in qaic_mmap_bo_ioctl()
739 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in qaic_mmap_bo_ioctl()
740 	if (qdev->in_reset) {  in qaic_mmap_bo_ioctl()
758 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_mmap_bo_ioctl()
836 static int qaic_prepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo,  in qaic_prepare_export_bo()  argument
844 	ret = dma_map_sgtable(&qdev->pdev->dev, bo->sgt, hdr->dir, 0);  in qaic_prepare_export_bo()
851 static int qaic_prepare_bo(struct qaic_device *qdev, struct qaic_bo *bo,  in qaic_prepare_bo()  argument
859 		ret = qaic_prepare_export_bo(qdev, bo, hdr);  in qaic_prepare_bo()
874 static void qaic_unprepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo)  in qaic_unprepare_export_bo()  argument
876 	dma_unmap_sgtable(&qdev->pdev->dev, bo->sgt, bo->dir, 0);  in qaic_unprepare_export_bo()
879 static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo)  in qaic_unprepare_bo()  argument
884 		qaic_unprepare_export_bo(qdev, bo);  in qaic_unprepare_bo()
897 static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo,  in qaic_attach_slicing_bo()  argument
904 		ret = qaic_map_one_slice(qdev, bo, &slice_ent[i]);  in qaic_attach_slicing_bo()
911 	if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) {  in qaic_attach_slicing_bo()
918 	list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists);  in qaic_attach_slicing_bo()
930 	struct qaic_device *qdev;  in qaic_attach_slice_bo_ioctl()  local
960 	qdev = usr->qddev->qdev;  in qaic_attach_slice_bo_ioctl()
961 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in qaic_attach_slice_bo_ioctl()
962 	if (qdev->in_reset) {  in qaic_attach_slice_bo_ioctl()
967 	if (args->hdr.dbc_id >= qdev->num_dbc) {  in qaic_attach_slice_bo_ioctl()
986 	ret = qaic_validate_req(qdev, slice_ent, args->hdr.count, args->hdr.size);  in qaic_attach_slice_bo_ioctl()
1003 	dbc = &qdev->dbc[args->hdr.dbc_id];  in qaic_attach_slice_bo_ioctl()
1010 	ret = qaic_prepare_bo(qdev, bo, &args->hdr);  in qaic_attach_slice_bo_ioctl()
1014 	ret = qaic_attach_slicing_bo(qdev, bo, &args->hdr, slice_ent);  in qaic_attach_slice_bo_ioctl()
1019 		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, args->hdr.dir);  in qaic_attach_slice_bo_ioctl()
1025 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_attach_slice_bo_ioctl()
1031 	qaic_unprepare_bo(qdev, bo);  in qaic_attach_slice_bo_ioctl()
1039 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_attach_slice_bo_ioctl()
1045 static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id,  in copy_exec_reqs()  argument
1048 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in copy_exec_reqs()
1086 static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice,  in copy_partial_exec_reqs()  argument
1089 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in copy_partial_exec_reqs()
1161 static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *file_priv,  in send_bo_list_to_device()  argument
1228 				ret = copy_partial_exec_reqs(qdev, slice,  in send_bo_list_to_device()
1232 				ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);  in send_bo_list_to_device()
1242 		dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir);  in send_bo_list_to_device()
1257 		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);  in send_bo_list_to_device()
1300 	struct qaic_device *qdev;  in __qaic_execute_bo_ioctl()  local
1338 	qdev = usr->qddev->qdev;  in __qaic_execute_bo_ioctl()
1339 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in __qaic_execute_bo_ioctl()
1340 	if (qdev->in_reset) {  in __qaic_execute_bo_ioctl()
1345 	if (args->hdr.dbc_id >= qdev->num_dbc) {  in __qaic_execute_bo_ioctl()
1350 	dbc = &qdev->dbc[args->hdr.dbc_id];  in __qaic_execute_bo_ioctl()
1369 	ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc,  in __qaic_execute_bo_ioctl()
1387 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in __qaic_execute_bo_ioctl()
1485 		if (dbc->qdev->in_reset) {  in irq_polling_work()
1529 	struct qaic_device *qdev;  in dbc_irq_threaded_fn()  local
1545 	qdev = dbc->qdev;  in dbc_irq_threaded_fn()
1582 			pci_dbg(qdev->pdev, "req_id %d failed with status %d\n", req_id, status);  in dbc_irq_threaded_fn()
1603 			dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);  in dbc_irq_threaded_fn()
1656 	struct qaic_device *qdev;  in qaic_wait_bo_ioctl()  local
1673 	qdev = usr->qddev->qdev;  in qaic_wait_bo_ioctl()
1674 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in qaic_wait_bo_ioctl()
1675 	if (qdev->in_reset) {  in qaic_wait_bo_ioctl()
1680 	if (args->dbc_id >= qdev->num_dbc) {  in qaic_wait_bo_ioctl()
1685 	dbc = &qdev->dbc[args->dbc_id];  in qaic_wait_bo_ioctl()
1718 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_wait_bo_ioctl()
1730 	struct qaic_device *qdev;  in qaic_perf_stats_bo_ioctl()  local
1742 	qdev = usr->qddev->qdev;  in qaic_perf_stats_bo_ioctl()
1743 	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);  in qaic_perf_stats_bo_ioctl()
1744 	if (qdev->in_reset) {  in qaic_perf_stats_bo_ioctl()
1749 	if (args->hdr.dbc_id >= qdev->num_dbc) {  in qaic_perf_stats_bo_ioctl()
1796 	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);  in qaic_perf_stats_bo_ioctl()
1802 static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc)  in empty_xfer_list()  argument
1813 		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);  in empty_xfer_list()
1821 int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)  in disable_dbc()  argument
1823 	if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle)  in disable_dbc()
1826 	qdev->dbc[dbc_id].usr = NULL;  in disable_dbc()
1827 	synchronize_srcu(&qdev->dbc[dbc_id].ch_lock);  in disable_dbc()
1839 void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)  in enable_dbc()  argument
1841 	qdev->dbc[dbc_id].usr = usr;  in enable_dbc()
1844 void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id)  in wakeup_dbc()  argument
1846 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in wakeup_dbc()
1849 	empty_xfer_list(qdev, dbc);  in wakeup_dbc()
1855 	empty_xfer_list(qdev, dbc);  in wakeup_dbc()
1858 void release_dbc(struct qaic_device *qdev, u32 dbc_id)  in release_dbc()  argument
1864 	dbc = &qdev->dbc[dbc_id];  in release_dbc()
1868 	wakeup_dbc(qdev, dbc_id);  in release_dbc()
1870 	dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr);  in release_dbc()