911 	if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) {  in qaic_attach_slicing_bo()
918 	list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists);  in qaic_attach_slicing_bo()
928 	struct dma_bridge_chan	*dbc;  in qaic_attach_slice_bo_ioctl()  local
1003 	dbc = &qdev->dbc[args->hdr.dbc_id];  in qaic_attach_slice_bo_ioctl()
1004 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in qaic_attach_slice_bo_ioctl()
1005 	if (dbc->usr != usr) {  in qaic_attach_slice_bo_ioctl()
1021 	bo->dbc = dbc;  in qaic_attach_slice_bo_ioctl()
1022 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in qaic_attach_slice_bo_ioctl()
1033 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in qaic_attach_slice_bo_ioctl()
1048 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in copy_exec_reqs()  local
1055 		avail += dbc->nelem;  in copy_exec_reqs()
1062 	if (tail + slice->nents > dbc->nelem) {  in copy_exec_reqs()
1063 		avail = dbc->nelem - tail;  in copy_exec_reqs()
1065 		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,  in copy_exec_reqs()
1070 			memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail);  in copy_exec_reqs()
1072 		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,  in copy_exec_reqs()
1076 	*ptail = (tail + slice->nents) % dbc->nelem;  in copy_exec_reqs()
1089 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in copy_partial_exec_reqs()  local
1102 		avail += dbc->nelem;  in copy_partial_exec_reqs()
1126 		if (tail + first_n > dbc->nelem) {  in copy_partial_exec_reqs()
1127 			avail = dbc->nelem - tail;  in copy_partial_exec_reqs()
1129 			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,  in copy_partial_exec_reqs()
1134 				memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail);  in copy_partial_exec_reqs()
1136 			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,  in copy_partial_exec_reqs()
1144 	last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size();  in copy_partial_exec_reqs()
1156 	*ptail = (tail + first_n + 1) % dbc->nelem;  in copy_partial_exec_reqs()
1163 				  bool is_partial, struct dma_bridge_chan *dbc, u32 head,  in send_bo_list_to_device()  argument
1199 		spin_lock_irqsave(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1203 			spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1208 		bo->req_id = dbc->next_req_id++;  in send_bo_list_to_device()
1230 							     dbc->id, head, tail);  in send_bo_list_to_device()
1232 				ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);  in send_bo_list_to_device()
1235 				spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1240 		list_add_tail(&bo->xfer_list, &dbc->xfer_list);  in send_bo_list_to_device()
1241 		spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1251 		spin_lock_irqsave(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1252 		bo = list_last_entry(&dbc->xfer_list, struct qaic_bo, xfer_list);  in send_bo_list_to_device()
1256 		spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in send_bo_list_to_device()
1298 	struct dma_bridge_chan *dbc;  in __qaic_execute_bo_ioctl()  local
1350 	dbc = &qdev->dbc[args->hdr.dbc_id];  in __qaic_execute_bo_ioctl()
1352 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in __qaic_execute_bo_ioctl()
1353 	if (!dbc->usr || dbc->usr->handle != usr->handle) {  in __qaic_execute_bo_ioctl()
1358 	head = readl(dbc->dbc_base + REQHP_OFF);  in __qaic_execute_bo_ioctl()
1359 	tail = readl(dbc->dbc_base + REQTP_OFF);  in __qaic_execute_bo_ioctl()
1367 	queue_level = head <= tail ? tail - head : dbc->nelem - (head - tail);  in __qaic_execute_bo_ioctl()
1369 	ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc,  in __qaic_execute_bo_ioctl()
1376 	writel(tail, dbc->dbc_base + REQTP_OFF);  in __qaic_execute_bo_ioctl()
1382 		schedule_work(&dbc->poll_work);  in __qaic_execute_bo_ioctl()
1385 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in __qaic_execute_bo_ioctl()
1440 	struct dma_bridge_chan *dbc = data;  in dbc_irq_handler()  local
1445 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in dbc_irq_handler()
1447 	if (!dbc->usr) {  in dbc_irq_handler()
1448 		srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_handler()
1452 	head = readl(dbc->dbc_base + RSPHP_OFF);  in dbc_irq_handler()
1454 		srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_handler()
1458 	tail = readl(dbc->dbc_base + RSPTP_OFF);  in dbc_irq_handler()
1460 		srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_handler()
1465 		srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_handler()
1470 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_handler()
1476 	struct dma_bridge_chan *dbc = container_of(work, struct dma_bridge_chan,  poll_work);  in irq_polling_work()  local
1482 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in irq_polling_work()
1485 		if (dbc->qdev->in_reset) {  in irq_polling_work()
1486 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1489 		if (!dbc->usr) {  in irq_polling_work()
1490 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1493 		spin_lock_irqsave(&dbc->xfer_lock, flags);  in irq_polling_work()
1494 		if (list_empty(&dbc->xfer_list)) {  in irq_polling_work()
1495 			spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in irq_polling_work()
1496 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1499 		spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in irq_polling_work()
1501 		head = readl(dbc->dbc_base + RSPHP_OFF);  in irq_polling_work()
1503 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1507 		tail = readl(dbc->dbc_base + RSPTP_OFF);  in irq_polling_work()
1509 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1514 			irq_wake_thread(dbc->irq, dbc);  in irq_polling_work()
1515 			srcu_read_unlock(&dbc->ch_lock, rcu_id);  in irq_polling_work()
1526 	struct dma_bridge_chan *dbc = data;  in dbc_irq_threaded_fn()  local
1539 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in dbc_irq_threaded_fn()
1541 	head = readl(dbc->dbc_base + RSPHP_OFF);  in dbc_irq_threaded_fn()
1545 	qdev = dbc->qdev;  in dbc_irq_threaded_fn()
1557 	if (!dbc->usr)  in dbc_irq_threaded_fn()
1560 	tail = readl(dbc->dbc_base + RSPTP_OFF);  in dbc_irq_threaded_fn()
1578 		rsp = dbc->rsp_q_base + head * sizeof(*rsp);  in dbc_irq_threaded_fn()
1583 		spin_lock_irqsave(&dbc->xfer_lock, flags);  in dbc_irq_threaded_fn()
1590 		list_for_each_entry_safe(bo, i, &dbc->xfer_list, xfer_list) {  in dbc_irq_threaded_fn()
1612 		spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in dbc_irq_threaded_fn()
1613 		head = (head + 1) % dbc->nelem;  in dbc_irq_threaded_fn()
1620 	writel(head, dbc->dbc_base + RSPHP_OFF);  in dbc_irq_threaded_fn()
1629 		schedule_work(&dbc->poll_work);  in dbc_irq_threaded_fn()
1631 	tail = readl(dbc->dbc_base + RSPTP_OFF);  in dbc_irq_threaded_fn()
1637 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_threaded_fn()
1641 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in dbc_irq_threaded_fn()
1645 		schedule_work(&dbc->poll_work);  in dbc_irq_threaded_fn()
1654 	struct dma_bridge_chan *dbc;  in qaic_wait_bo_ioctl()  local
1685 	dbc = &qdev->dbc[args->dbc_id];  in qaic_wait_bo_ioctl()
1687 	rcu_id = srcu_read_lock(&dbc->ch_lock);  in qaic_wait_bo_ioctl()
1688 	if (dbc->usr != usr) {  in qaic_wait_bo_ioctl()
1710 	if (!dbc->usr)  in qaic_wait_bo_ioctl()
1716 	srcu_read_unlock(&dbc->ch_lock, rcu_id);  in qaic_wait_bo_ioctl()
1802 static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc)  in empty_xfer_list()  argument
1807 	spin_lock_irqsave(&dbc->xfer_lock, flags);  in empty_xfer_list()
1808 	while (!list_empty(&dbc->xfer_list)) {  in empty_xfer_list()
1809 		bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list);  in empty_xfer_list()
1812 		spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in empty_xfer_list()
1816 		spin_lock_irqsave(&dbc->xfer_lock, flags);  in empty_xfer_list()
1818 	spin_unlock_irqrestore(&dbc->xfer_lock, flags);  in empty_xfer_list()
1823 	if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle)  in disable_dbc()
1826 	qdev->dbc[dbc_id].usr = NULL;  in disable_dbc()
1827 	synchronize_srcu(&qdev->dbc[dbc_id].ch_lock);  in disable_dbc()
1841 	qdev->dbc[dbc_id].usr = usr;  in enable_dbc()
1846 	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];  in wakeup_dbc()  local
1848 	dbc->usr = NULL;  in wakeup_dbc()
1849 	empty_xfer_list(qdev, dbc);  in wakeup_dbc()
1850 	synchronize_srcu(&dbc->ch_lock);  in wakeup_dbc()
1855 	empty_xfer_list(qdev, dbc);  in wakeup_dbc()
1862 	struct dma_bridge_chan *dbc;  in release_dbc()  local
1864 	dbc = &qdev->dbc[dbc_id];  in release_dbc()
1865 	if (!dbc->in_use)  in release_dbc()
1870 	dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr);  in release_dbc()
1871 	dbc->total_size = 0;  in release_dbc()
1872 	dbc->req_q_base = NULL;  in release_dbc()
1873 	dbc->dma_addr = 0;  in release_dbc()
1874 	dbc->nelem = 0;  in release_dbc()
1875 	dbc->usr = NULL;  in release_dbc()
1877 	list_for_each_entry_safe(bo, bo_temp, &dbc->bo_lists, bo_list) {  in release_dbc()
1884 		bo->dbc = NULL;  in release_dbc()
1898 	dbc->in_use = false;  in release_dbc()
1899 	wake_up(&dbc->dbc_release);  in release_dbc()